def __init__(self, env, results_path, tok, episode_len=20):
super(Seq2SeqAgent, self).__init__(env, results_path)
self.tok = tok
self.episode_len = episode_len
self.feature_size = self.env.feature_size
# Models
if args.vlnbert == 'oscar':
self.vln_bert = model_OSCAR.VLNBERT(
feature_size=self.feature_size + args.angle_feat_size).cuda()
self.critic = model_OSCAR.Critic().cuda()
elif args.vlnbert == 'prevalent':
self.vln_bert = model_PREVALENT.VLNBERT(
feature_size=self.feature_size + args.angle_feat_size).cuda()
self.critic = model_PREVALENT.Critic().cuda()
self.models = (self.vln_bert, self.critic)
# Optimizers
self.vln_bert_optimizer = args.optimizer(self.vln_bert.parameters(),
lr=args.lr)
self.critic_optimizer = args.optimizer(self.critic.parameters(),
lr=args.lr)
self.optimizers = (self.vln_bert_optimizer, self.critic_optimizer)
# Evaluations
self.losses = []
self.criterion = nn.CrossEntropyLoss(ignore_index=args.ignoreid,
size_average=False)
self.ndtw_criterion = utils.ndtw_initialize()
# Logs
sys.stdout.flush()
self.logs = defaultdict(list)
def __init__(self, env, results_path, tok, episode_len=20):
super(Seq2SeqAgent, self).__init__(env, results_path)
self.tok = tok
self.episode_len = episode_len
self.feature_size = self.env.feature_size
# Models
enc_hidden_size = args.rnn_dim//2 if args.bidir else args.rnn_dim
self.encoder = model.EncoderLSTM(tok.vocab_size(), args.wemb, enc_hidden_size, padding_idx,
args.dropout, bidirectional=args.bidir).cuda()
self.decoder = model.AttnDecoderLSTM(args.aemb, args.rnn_dim, args.dropout, feature_size=self.feature_size + args.angle_feat_size).cuda()
self.critic = model.Critic().cuda()
self.models = (self.encoder, self.decoder, self.critic)
# Optimizers
self.encoder_optimizer = args.optimizer(self.encoder.parameters(), lr=args.lr)
self.decoder_optimizer = args.optimizer(self.decoder.parameters(), lr=args.lr)
self.critic_optimizer = args.optimizer(self.critic.parameters(), lr=args.lr)
self.optimizers = (self.encoder_optimizer, self.decoder_optimizer, self.critic_optimizer)
# Evaluations
self.losses = []
self.criterion = nn.CrossEntropyLoss(ignore_index=args.ignoreid, size_average=False)
# Logs
sys.stdout.flush()
self.logs = defaultdict(list)
def __init__(self, env, listener, tok):
self.env = env
self.feature_size = self.env.feature_size
self.tok = tok
self.tok.finalize()
self.listener = listener
# Model
print("VOCAB_SIZE", self.tok.vocab_size())
self.encoder = model.SpeakerEncoder(self.feature_size +
args.angle_feat_size,
args.rnn_dim,
args.dropout,
bidirectional=args.bidir).cuda()
self.decoder = model.SpeakerDecoder(self.tok.vocab_size(), args.wemb,
self.tok.word_to_index['<PAD>'],
args.rnn_dim, args.dropout).cuda()
self.encoder_optimizer = args.optimizer(self.encoder.parameters(),
lr=args.lr)
self.decoder_optimizer = args.optimizer(self.decoder.parameters(),
lr=args.lr)
# Evaluation
self.softmax_loss = torch.nn.CrossEntropyLoss(
ignore_index=self.tok.word_to_index['<PAD>'])
# Will be used in beam search
self.nonreduced_softmax_loss = torch.nn.CrossEntropyLoss(
ignore_index=self.tok.word_to_index['<PAD>'],
size_average=False,
reduce=False)
def __init__(self, env, results_path, tok, episode_len=20):
super(Seq2SeqAgent, self).__init__(env, results_path)
self.tok = tok
self.episode_len = episode_len
self.feature_size = self.env.feature_size
# Models
self.glove_dim = 300
with open('img_features/objects/object_vocab.txt', 'r') as f_ov:
self.obj_vocab = [k.strip() for k in f_ov.readlines()]
glove_matrix = get_glove_matrix(self.obj_vocab, self.glove_dim)
self.objencoder = ObjEncoder(glove_matrix.size(0), glove_matrix.size(1), glove_matrix).cuda()
enc_hidden_size = args.rnn_dim//2 if args.bidir else args.rnn_dim
self.encoder = model.EncoderLSTM(tok.vocab_size(), args.wemb, enc_hidden_size, padding_idx,
args.dropout, bidirectional=args.bidir).cuda()
self.decoder = model.AttnDecoderLSTM_Graph(args.aemb, args.rnn_dim, args.dropout, feature_size=self.feature_size + args.angle_feat_size).cuda()
self.critic = model.Critic().cuda()
self.models = (self.encoder, self.decoder, self.critic)
# Optimizers
self.encoder_optimizer = args.optimizer(self.encoder.parameters(), lr=args.lr)
self.decoder_optimizer = args.optimizer(self.decoder.parameters(), lr=args.lr)
self.critic_optimizer = args.optimizer(self.critic.parameters(), lr=args.lr)
self.optimizers = (self.encoder_optimizer, self.decoder_optimizer, self.critic_optimizer)
# Evaluations
self.losses = []
self.criterion = nn.CrossEntropyLoss(ignore_index=args.ignoreid, size_average=False)
# Logs
sys.stdout.flush()
self.logs = defaultdict(list)
def __init__(self, folder="/", load=True):
# Datasets
if load:
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
folder=folder)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=128,
shuffle=False,
drop_last=False,
folder=folder,
nops=args.nops)
else:
self.valid_tuple = None
# Model
# self.model = VQAModel(self.train_tuple.dataset.num_answers)
is_cp = False
if "vqacpv2" in folder:
is_cp = True
if not is_cp:
self.model = VQAModel(3129)
else:
self.model = VQAModel(16039)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if load:
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.bert_type == 'ft':
bs_infer = 256
else:
bs_infer = 1024
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=bs_infer,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
print("args.lr is {0}".format(args.lr))
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
# if type(args.lr) == type("sdfg"):
# args.lr = float(args.lr)
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total,
schedule=args.lr_schedule,
args=args)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self):
self.train_tuple = get_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
valid_bsize = 2048 if args.multiGPU else 512
self.valid_tuple = get_tuple(args.valid,
bs=valid_bsize,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
self.model = GQAModel(self.train_tuple.dataset.num_answers)
self.momentum = 0.99997
self.siam_model = copy.deepcopy(self.model)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
self.siam_model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
load_lxmert_qa(args.load_lxmert_qa,
self.siam_model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
self.siam_model = self.siam_model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
self.siam_model.lxrt_encoder.multi_gpu()
# Losses and optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1)
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(list(self.model.parameters()), args.lr)
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self, env, listener, tok):
self.env = env
self.feature_size = self.env.feature_size
self.tok = tok
self.tok.finalize()
self.listener = listener
# Model
self.glove_dim = 300
with open(
'/students/u5399302/MatterportData/data/obj_stats2/objectvocab_filtered_dyno4.txt',
'r') as f_ov:
self.obj_vocab = [k.strip() for k in f_ov.readlines()]
glove_matrix = utils.get_glove_matrix(self.obj_vocab, self.glove_dim)
self.objencoder = ObjEncoder(glove_matrix.size(0),
glove_matrix.size(1),
glove_matrix).cuda()
print("VOCAB_SIZE", self.tok.vocab_size())
self.encoder = model.SpeakerEncoder(self.feature_size +
args.angle_feat_size,
args.rnn_dim,
args.dropout,
bidirectional=args.bidir).cuda()
self.decoder = model.SpeakerDecoder(self.tok.vocab_size(), args.wemb,
self.tok.word_to_index['<PAD>'],
args.rnn_dim, args.dropout).cuda()
self.encoder_optimizer = args.optimizer(self.encoder.parameters(),
lr=args.lr)
self.decoder_optimizer = args.optimizer(self.decoder.parameters(),
lr=args.lr)
# Evaluation
self.softmax_loss = torch.nn.CrossEntropyLoss(
ignore_index=self.tok.word_to_index['<PAD>'])
# Will be used in beam search
self.nonreduced_softmax_loss = torch.nn.CrossEntropyLoss(
ignore_index=self.tok.word_to_index['<PAD>'],
size_average=False,
reduce=False)
def setup_optimizers(self):
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(list(self.model.parameters()), args.lr)
def __init__(self, dataset):
# Built up the model
self.tok = dataset.tok
self.feature_size = 2048
if args.model == 'init':
self.encoder = SpeakerEncoder(self.feature_size).cuda()
ctx_size = self.feature_size
self.decoder = SpeakerDecoderTran(self.tok.vocab_size, ctx_size, heads=1).cuda()
elif args.model == 'heads':
self.encoder = MultiCtxEncoder(self.feature_size).cuda()
ctx_size = self.encoder.ctx_dim
self.decoder = MultiCtxDecoder(self.tok.vocab_size, ctx_size, heads=2).cuda()
elif args.model == 'crossatt':
self.encoder = CrossAttEncoder(self.feature_size).cuda()
ctx_size = self.encoder.ctx_dim
self.decoder = MultiCtxDecoder(self.tok.vocab_size, ctx_size, heads=2).cuda()
elif args.model == 'newheads':
self.encoder = MultiCtxEncoder(self.feature_size).cuda()
ctx_size = self.encoder.ctx_dim
self.decoder = NewCtxDecoder(self.tok.vocab_size, ctx_size, heads=2).cuda()
elif args.model == 'newcross':
self.encoder = NewAttEncoder(self.feature_size).cuda()
ctx_size = self.encoder.ctx_dim
self.decoder = NewCtxDecoder(self.tok.vocab_size, ctx_size, heads=2).cuda()
elif args.model == 'dynamic':
self.encoder = MultiCtxEncoder(self.feature_size).cuda()
ctx_size = self.encoder.ctx_dim
self.decoder = DynamicDecoderFC(self.tok.vocab_size, ctx_size).cuda()
elif args.model == 'dynamicmh':
self.encoder = MultiCtxEncoder(self.feature_size).cuda()
ctx_size = self.encoder.ctx_dim
self.decoder = DynamicDecoderMH(self.tok.vocab_size, ctx_size).cuda()
elif args.model == 'dmc':
self.encoder = MultiCtxEncoder(self.feature_size).cuda()
ctx_size = self.encoder.ctx_dim
self.decoder = DynamicDecoderMHC(self.tok.vocab_size, ctx_size).cuda()
if args.baseline == 'linear':
self.critic = LinearAct(args.hid_dim, 1).cuda()
# Optimizer
self.optim = args.optimizer(list(self.encoder.parameters()) + list(self.decoder.parameters()),
lr=args.lr)
# Logs
self.output = args.output
os.makedirs(self.output, exist_ok=True)
self.writer = SummaryWriter(log_dir=self.output) # Tensorboard summary writer
# Loss
self.softmax_loss = torch.nn.CrossEntropyLoss(ignore_index=self.tok.pad_id)
def __init__(self,folder="/",load=True):
# Datasets
if load:
self.train_tuple = get_data_tuple(
args.train, bs=args.batch_size, shuffle=True, drop_last=True,folder=folder
)
if args.valid != "":
self.valid_tuple = get_data_tuple(
args.valid, bs=128,
shuffle=False, drop_last=False, folder=folder,nops=args.nops
)
else:
self.valid_tuple = None
get_bias(self.train_tuple.dataset,self.valid_tuple.dataset)
# Model
# self.model = VQAModel(self.train_tuple.dataset.num_answers)
label2ans = json.load(open("/data/datasets/vqa_mutant/data/vqa/mutant_l2a/mutant_label2ans.json"))
self.model = VQAModel(len(label2ans))
self.debias = LearnedMixin(w=0.36,hid_dim=self.model.lxrt_encoder.dim)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa, self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
self.debias = self.debias.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if load :
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
valid_bsize = args.get("valid_batch_size", 16)
self.valid_tuple = get_data_tuple(args.valid,
bs=valid_bsize,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.get("load_lxmert_pretrain", None) is not None:
load_lxmert_from_pretrain_noqa(args.load_lxmert_pretrain,
self.model)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
self.model.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self,
env,
tok,
obs_dim,
latent_dim,
loss_type="mse",
view_num=args.view_num,
path_len=2):
super(BaseVAE, self).__init__()
self.env = env
self.tok = tok
self.obs_dim = obs_dim
self.latent_dim = latent_dim
self.path_len = path_len
self.view_num = view_num
self.loss_type = loss_type
self.encoder = StateEncoder(obs_dim, latent_dim)
self.decoder = StateDecoder(obs_dim,
latent_dim,
view_num,
path_len=path_len)
self.policy = PolicyDecoder(obs_dim, latent_dim, view_num, path_len)
self.unit_n = Normal(Variable(torch.zeros(1, latent_dim)).cuda(),
log_var=Variable(torch.zeros(1,
latent_dim)).cuda())
self.encoder_optimizer = args.optimizer(self.encoder.parameters(),
lr=args.lr)
self.decoder_optimizer = args.optimizer(self.decoder.parameters(),
lr=args.lr)
self.policy_optimizer = args.optimizer(self.policy.parameters(),
lr=args.lr)
self.vae_loss_weight = args.vae_loss_weight
self.vae_kl_weight = args.vae_kl_weight
# self.bc_weight = 100
self.vae_bc_weight = args.vae_bc_weight
self.iter = None # training iteration indicator
def __init__(self, env, results_path, tok, episode_len=20):
super(Seq2PolicyAgent, self).__init__(env, results_path)
self._iter = 0
self.tok = tok
self.episode_len = episode_len
self.feature_size = self.env.feature_size
# Models
enc_hidden_size = args.rnn_dim//2 if args.bidir else args.rnn_dim
self.encoder = model.EncoderLSTM(tok.vocab_size(), args.wemb, enc_hidden_size, padding_idx,
args.dropout, bidirectional=args.bidir).cuda()
if args.original_decoder:
self.decoder = model.AttnDecoderLSTM(args.aemb, args.rnn_dim, args.dropout, feature_size=self.feature_size + args.angle_feat_size).cuda()
else:
self.decoder = model.AttnPolicyLSTM(args.aemb, args.rnn_dim, args.dropout, feature_size=self.feature_size + args.angle_feat_size, latent_dim=args.vae_latent_dim).cuda()
if args.fix_vae:
print("fix the parameters in sub policy")
for param in self.decoder.policy.parameters():
param.requires_grad = False
self.critic = model.Critic().cuda()
self.models = (self.encoder, self.decoder, self.critic)
# Optimizers
self.encoder_optimizer = args.optimizer(self.encoder.parameters(), lr=args.lr)
self.decoder_optimizer = args.optimizer(self.decoder.parameters(), lr=args.lr)
self.critic_optimizer = args.optimizer(self.critic.parameters(), lr=args.lr)
self.optimizers = (self.encoder_optimizer, self.decoder_optimizer, self.critic_optimizer)
# Evaluations
self.losses = []
self.criterion = nn.CrossEntropyLoss(ignore_index=args.ignoreid, size_average=False)
# Logs
sys.stdout.flush()
self.logs = defaultdict(list)
def prepare_model(self):
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
def __init__(self):
self.train_tuple = get_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
valid_bsize = 2048 if args.multiGPU else 512
self.valid_tuple = get_tuple(args.valid,
bs=valid_bsize,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
self.model = NLVR2Model()
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
# GPU options
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
self.model = self.model.cuda()
# Losses and optimizer
self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1)
if "bert" in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(list(self.model.parameters()), args.lr)
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self, folder="/", load=True):
# Datasets
if load:
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
folder=folder)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=512,
shuffle=False,
drop_last=False,
folder=folder)
else:
self.valid_tuple = None
# Model
# self.model = VQAModel(self.train_tuple.dataset.num_answers)
self.model = VQAModel(len(self.train_tuple.dataset.label2ans),
fn_type=args.fn_type)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Load IndexList of Answer to Type Map
self.indexlist = json.load(
open(
"/data/datasets/vqa_mutant/data/vqa/mutant_l2a/mutant_merge_indexlist.json"
))
print("Length of Masks", len(self.indexlist), flush=True)
indextensor = torch.cuda.LongTensor(self.indexlist)
self.mask0 = torch.eq(indextensor, 0).float()
self.mask1 = torch.eq(indextensor, 1).float()
self.mask2 = torch.eq(indextensor, 2).float()
self.mask3 = torch.eq(indextensor, 3).float()
self.mask_cache = {}
# Loss and Optimizer
self.logsoftmax = nn.LogSoftmax()
self.sigmoid = nn.Sigmoid()
self.softmax = nn.Softmax()
self.bceloss = nn.BCELoss()
self.nllloss = nn.NLLLoss()
self.bce_loss = nn.BCEWithLogitsLoss()
self.ce_loss = nn.CrossEntropyLoss()
if load:
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self, attention=False):
# Datasets
print("Fetching data")
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
dataset_name="test")
print("Got data")
print("fetching val data")
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=args.batch_size,
shuffle=False,
drop_last=False,
dataset_name="test")
print("got data")
else:
self.valid_tuple = None
print("Got data")
# Model
print("Making model")
self.model = VQAModel(self.train_tuple.dataset.num_answers, attention)
print("Ready model")
# Print model info:
print("Num of answers:")
print(self.train_tuple.dataset.num_answers)
# print("Model info:")
# print(self.model)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
def __init__(self, folder="/", load=True):
# Datasets
if load:
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True,
folder=folder)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=128,
shuffle=False,
drop_last=False,
folder=folder,
nops=args.nops)
else:
self.valid_tuple = None
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers)
# is_cp=False
# if "vqacpv2" in folder:
# is_cp=True
# if not is_cp:
# self.model = VQAModel(3129)
# else:
# self.model = VQAModel(16039)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
ans_embed = np.load(
"/data/datasets/vqa_mutant/data/vqa/mutant_l2a/answer_embs.npy"
) + 1e-8
ans_embed = torch.tensor(ans_embed).cuda()
self.ans_embed = torch.nn.functional.normalize(ans_embed, dim=1)
self.embed_cache = {}
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if load:
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
self.cos = nn.CosineSimilarity()
def __init__(self):
self.train_type = args.train_type
self.device = torch.device(args.device)
# Dataloaders for train and val set
if not args.test:
self.valid_tuple = get_data_tuple(args.valid,
bs=args.batch_size,
shuffle=False,
drop_last=False)
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
num_answers = self.train_tuple.dataset.num_answers
file_name = args.train
log_str = f"\n{ctime()} || Loaded train set of size {len(self.train_tuple[0])} and val set of size {len(self.valid_tuple[0])}."
else:
self.test_tuple = get_data_tuple(args.test,
bs=args.batch_size,
shuffle=False,
drop_last=False)
num_answers = self.test_tuple.dataset.num_answers
file_name = args.test
log_str = (
f"\n{ctime()} || Loaded test set of size {len(self.test_tuple[0])}."
)
# get dataset name
self.dtype = args.task
# Model
self.model = eUGModel(self.train_type, num_answers, self.dtype,
args.model)
# Load pre-trained weights
if self.train_type == "expl" and args.bb_path is not None:
self.model.load_state_dict(torch.load(args.bb_path))
# freeze backbone
for p, n in self.model.named_parameters():
if "decoder.model.transformer" not in p:
n.requires_grad = False
elif args.load_pretrained is not None:
self.model.encoder.load(args.load_pretrained)
self.model = self.model.to(self.device)
# Loss and Optimizer
if not args.test:
if self.dtype == "vqa_x":
self.loss_func = nn.BCEWithLogitsLoss()
else:
self.loss_func = nn.CrossEntropyLoss()
batch_per_epoch = len(self.train_tuple.loader) / args.grad_accum
t_total = int(batch_per_epoch * args.epochs)
if "bert" in args.optim:
print("BertAdam Total Iters: %d" % t_total)
from src.optimization import BertAdam
self.optim = BertAdam(
list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total,
)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
self.scheduler = get_linear_schedule_with_warmup(
self.optim,
num_warmup_steps=args.warmup_steps,
num_training_steps=t_total,
)
self.grad_accum = args.grad_accum
# Output Directory
self.output = args.output
self.save_steps = args.save_steps
os.makedirs(self.output, exist_ok=True)
# print logs
log_str += f"\n{ctime()} || Model loaded. Batch size {args.batch_size*args.grad_accum} | lr {args.lr} | task: {self.dtype} | type: {self.train_type}."
print_log(args, log_str)
def __init__(self):
self.train_tuple = get_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
valid_bsize = 2048 if args.multiGPU else 512
self.valid_tuple = get_tuple(args.valid,
bs=valid_bsize,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
self.model = GQAModel(self.train_tuple.dataset.num_answers)
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
self.new_ans_label = load_lxmert_qa(
args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
# Losses and optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1)
# self.KL_loss = nn.KLDivLoss(reduction='none')
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(list(self.model.parameters()), args.lr)
self.output = args.output
os.makedirs(self.output, exist_ok=True)
# Tensorboard
self.boards_dir = os.path.join('boards', self.output)
if not os.path.exists(self.boards_dir):
os.makedirs(self.boards_dir)
self.writerTbrd = SummaryWriter(self.boards_dir)
# get Glove projection for all answers
if args.answer_loss == 'glove':
path_glove = './data/GloVe/GloVeDict.pkl'
with open(path_glove, 'rb') as f:
glove_dic = pickle.load(f)
glove_dim = glove_dic['the'].shape[-1]
print("Loading Glove%d answer's vector" % glove_dim)
self.labelans2glove = []
self.valid_ans_embed = [1] * len(
self.train_tuple.dataset.label2ans)
for label, ans in enumerate(self.train_tuple.dataset.label2ans):
ans = ans.split(' ')
glove_ans = []
for w in ans:
#print(w)
try:
glove_ans.append(glove_dic[w])
except KeyError:
#print('Full ans: %s' % ans)
#input(' ')
self.valid_ans_embed[label] = 0
glove_ans.append(np.zeros(glove_dim))
#print(glove_ans)
glove_ans = torch.tensor(glove_ans).mean(-2)
self.labelans2glove.append(torch.tensor(glove_ans))
#print(self.labelans2glove)
print(
'Ratio of valid ans embedding: %f' %
(float(sum(self.valid_ans_embed)) / len(self.valid_ans_embed)))
self.labelans2glove = torch.stack(
self.labelans2glove).float().cuda()
self.cosineSim = torch.nn.CosineSimilarity(dim=1, eps=1e-08)
def finetune(self, train_tuple, eval_tuple):
# log
output_1 = os.path.join(self.output, 'finetune_1')
os.makedirs(output_1, exist_ok=True)
output_2 = os.path.join(self.output, 'finetune_2')
os.makedirs(output_2, exist_ok=True)
# Tensorboard
boards_dir_1 = os.path.join(self.boards_dir, 'finetune_1')
if not os.path.exists(boards_dir_1):
os.makedirs(boards_dir_1)
boards_dir_2 = os.path.join(self.boards_dir, 'finetune_2')
if not os.path.exists(boards_dir_2):
os.makedirs(boards_dir_2)
# Params
lr_1 = args.lr
lr_2 = args.lr / 10
epochs_1 = 4 #int(args.epochs / 3)
epochs_2 = args.epochs - epochs_1
# Step 0: evaluate pretraining
if self.valid_tuple is not None: # Do Validation
valid_score = self.evaluate(eval_tuple)
print("Before finetune: Valid %0.2f\n" % (valid_score * 100.))
# Step 0.1: finetune new ans only
# new_ans_params = []
# for name, p in self.model.named_parameters():
# if "logit_fc.3" in name:
# for idx in range(p.size(0)):
# if idx in self.new_ans_label:
# new_ans_params.append({'params': p[idx]})
# args.epochs = epochs_0
# from lxrt.optimization import BertAdam
# self.optim = BertAdam(new_ans_params,
# lr=lr_1,
# warmup=0.0,
# t_total=-1)
# print('### Start finetuning new ans...')
# self.train(train_tuple, eval_tuple)
# First step, only updates answer head
#self.optim = torch.optim.Adamax(list(self.model.parameters()), lr_1)
#self.optim = torch.optim.SGD(list(self.model.parameters()), lr_1)
args.epochs = epochs_1
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * epochs_1)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(
list(self.model.parameters()),
lr=lr_1,
warmup=0.0, #!0.034
t_total=-1)
# loaded_optim = torch.load("%s_LXRT.pth" % args.load_lxmert_qa)['optimizer']
# self.optim.load_state_dict(loaded_optim)
# for group in loaded_optim.param_groups:
# for p in group['params']:
# if p in loaded_optim['state']:
# self.optim.state[p] = loaded_optim.state[p]
self.writerTbrd = SummaryWriter(boards_dir_1)
self.output = output_1
for name, p in self.model.named_parameters():
if "logit_fc" in name:
p.requires_grad = True
else:
p.requires_grad = False
print('### Start finetuning step 1...')
self.train(train_tuple, eval_tuple)
# Second step, finetune all
for name, p in self.model.named_parameters():
p.requires_grad = True
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * epochs_2)
print("Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=lr_2,
warmup=0.1,
t_total=t_total,
lr_min=1e-7)
else:
self.optim = args.optimizer(list(self.model.parameters()), lr_2)
args.epochs = epochs_2
self.writerTbrd = SummaryWriter(boards_dir_2)
self.output = output_2
print('### Start finetuning step 2...')
self.train(train_tuple, eval_tuple)
def __init__(self):
# Datasets
self.train_tuple = get_data_tuple(args.train,
bs=args.batch_size,
shuffle=True,
drop_last=True)
if args.valid != "":
self.valid_tuple = get_data_tuple(args.valid,
bs=1024,
shuffle=False,
drop_last=False)
else:
self.valid_tuple = None
# Model
self.model = VQAModel(self.train_tuple.dataset.num_answers,
finetune_strategy=args.finetune_strategy)
# if finetune strategy is spottune
if args.finetune_strategy in PolicyStrategies:
self.policy_model = PolicyLXRT(
PolicyStrategies[args.finetune_strategy])
# Load pre-trained weights
if args.load_lxmert is not None:
self.model.lxrt_encoder.load(args.load_lxmert)
if args.load_lxmert_qa is not None:
load_lxmert_qa(args.load_lxmert_qa,
self.model,
label2ans=self.train_tuple.dataset.label2ans)
# GPU options
self.model = self.model.cuda()
if args.finetune_strategy in PolicyStrategies:
self.policy_model = self.policy_model.cuda()
if args.multiGPU:
self.model.lxrt_encoder.multi_gpu()
self.policy_model.policy_lxrt_encoder.multi_gpu()
# Loss and Optimizer
self.bce_loss = nn.BCEWithLogitsLoss()
if 'bert' in args.optim:
batch_per_epoch = len(self.train_tuple.loader)
t_total = int(batch_per_epoch * args.epochs)
print("BertAdam Total Iters: %d" % t_total)
from lxrt.optimization import BertAdam
self.optim = BertAdam(list(self.model.parameters()),
lr=args.lr,
warmup=0.1,
t_total=t_total)
else:
self.optim = args.optimizer(self.model.parameters(), args.lr)
# Optimizer for policy net
if args.finetune_strategy in PolicyStrategies:
self.policy_optim = args.policy_optimizer(
self.policy_model.parameters(), args.policy_lr)
# Output Directory
self.output = args.output
os.makedirs(self.output, exist_ok=True)
