def __init__(self, env, results_path, tok, episode_len=20): super(Seq2SeqAgent, self).__init__(env, results_path) self.tok = tok self.episode_len = episode_len self.feature_size = self.env.feature_size # Models if args.vlnbert == 'oscar': self.vln_bert = model_OSCAR.VLNBERT( feature_size=self.feature_size + args.angle_feat_size).cuda() self.critic = model_OSCAR.Critic().cuda() elif args.vlnbert == 'prevalent': self.vln_bert = model_PREVALENT.VLNBERT( feature_size=self.feature_size + args.angle_feat_size).cuda() self.critic = model_PREVALENT.Critic().cuda() self.models = (self.vln_bert, self.critic) # Optimizers self.vln_bert_optimizer = args.optimizer(self.vln_bert.parameters(), lr=args.lr) self.critic_optimizer = args.optimizer(self.critic.parameters(), lr=args.lr) self.optimizers = (self.vln_bert_optimizer, self.critic_optimizer) # Evaluations self.losses = [] self.criterion = nn.CrossEntropyLoss(ignore_index=args.ignoreid, size_average=False) self.ndtw_criterion = utils.ndtw_initialize() # Logs sys.stdout.flush() self.logs = defaultdict(list)
def __init__(self, env, results_path, tok, episode_len=20): super(Seq2SeqAgent, self).__init__(env, results_path) self.tok = tok self.episode_len = episode_len self.feature_size = self.env.feature_size # Models enc_hidden_size = args.rnn_dim//2 if args.bidir else args.rnn_dim self.encoder = model.EncoderLSTM(tok.vocab_size(), args.wemb, enc_hidden_size, padding_idx, args.dropout, bidirectional=args.bidir).cuda() self.decoder = model.AttnDecoderLSTM(args.aemb, args.rnn_dim, args.dropout, feature_size=self.feature_size + args.angle_feat_size).cuda() self.critic = model.Critic().cuda() self.models = (self.encoder, self.decoder, self.critic) # Optimizers self.encoder_optimizer = args.optimizer(self.encoder.parameters(), lr=args.lr) self.decoder_optimizer = args.optimizer(self.decoder.parameters(), lr=args.lr) self.critic_optimizer = args.optimizer(self.critic.parameters(), lr=args.lr) self.optimizers = (self.encoder_optimizer, self.decoder_optimizer, self.critic_optimizer) # Evaluations self.losses = [] self.criterion = nn.CrossEntropyLoss(ignore_index=args.ignoreid, size_average=False) # Logs sys.stdout.flush() self.logs = defaultdict(list)
def __init__(self, env, listener, tok): self.env = env self.feature_size = self.env.feature_size self.tok = tok self.tok.finalize() self.listener = listener # Model print("VOCAB_SIZE", self.tok.vocab_size()) self.encoder = model.SpeakerEncoder(self.feature_size + args.angle_feat_size, args.rnn_dim, args.dropout, bidirectional=args.bidir).cuda() self.decoder = model.SpeakerDecoder(self.tok.vocab_size(), args.wemb, self.tok.word_to_index['<PAD>'], args.rnn_dim, args.dropout).cuda() self.encoder_optimizer = args.optimizer(self.encoder.parameters(), lr=args.lr) self.decoder_optimizer = args.optimizer(self.decoder.parameters(), lr=args.lr) # Evaluation self.softmax_loss = torch.nn.CrossEntropyLoss( ignore_index=self.tok.word_to_index['<PAD>']) # Will be used in beam search self.nonreduced_softmax_loss = torch.nn.CrossEntropyLoss( ignore_index=self.tok.word_to_index['<PAD>'], size_average=False, reduce=False)
def __init__(self, env, results_path, tok, episode_len=20): super(Seq2SeqAgent, self).__init__(env, results_path) self.tok = tok self.episode_len = episode_len self.feature_size = self.env.feature_size # Models self.glove_dim = 300 with open('img_features/objects/object_vocab.txt', 'r') as f_ov: self.obj_vocab = [k.strip() for k in f_ov.readlines()] glove_matrix = get_glove_matrix(self.obj_vocab, self.glove_dim) self.objencoder = ObjEncoder(glove_matrix.size(0), glove_matrix.size(1), glove_matrix).cuda() enc_hidden_size = args.rnn_dim//2 if args.bidir else args.rnn_dim self.encoder = model.EncoderLSTM(tok.vocab_size(), args.wemb, enc_hidden_size, padding_idx, args.dropout, bidirectional=args.bidir).cuda() self.decoder = model.AttnDecoderLSTM_Graph(args.aemb, args.rnn_dim, args.dropout, feature_size=self.feature_size + args.angle_feat_size).cuda() self.critic = model.Critic().cuda() self.models = (self.encoder, self.decoder, self.critic) # Optimizers self.encoder_optimizer = args.optimizer(self.encoder.parameters(), lr=args.lr) self.decoder_optimizer = args.optimizer(self.decoder.parameters(), lr=args.lr) self.critic_optimizer = args.optimizer(self.critic.parameters(), lr=args.lr) self.optimizers = (self.encoder_optimizer, self.decoder_optimizer, self.critic_optimizer) # Evaluations self.losses = [] self.criterion = nn.CrossEntropyLoss(ignore_index=args.ignoreid, size_average=False) # Logs sys.stdout.flush() self.logs = defaultdict(list)
def __init__(self, folder="/", load=True): # Datasets if load: self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True, folder=folder) if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=128, shuffle=False, drop_last=False, folder=folder, nops=args.nops) else: self.valid_tuple = None # Model # self.model = VQAModel(self.train_tuple.dataset.num_answers) is_cp = False if "vqacpv2" in folder: is_cp = True if not is_cp: self.model = VQAModel(3129) else: self.model = VQAModel(16039) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if load: if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self): # Datasets self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) if args.bert_type == 'ft': bs_infer = 256 else: bs_infer = 1024 if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=bs_infer, shuffle=False, drop_last=False) else: self.valid_tuple = None print("args.lr is {0}".format(args.lr)) # Model self.model = VQAModel(self.train_tuple.dataset.num_answers) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if 'bert' in args.optim: # if type(args.lr) == type("sdfg"): # args.lr = float(args.lr) batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total, schedule=args.lr_schedule, args=args) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self): self.train_tuple = get_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) if args.valid != "": valid_bsize = 2048 if args.multiGPU else 512 self.valid_tuple = get_tuple(args.valid, bs=valid_bsize, shuffle=False, drop_last=False) else: self.valid_tuple = None self.model = GQAModel(self.train_tuple.dataset.num_answers) self.momentum = 0.99997 self.siam_model = copy.deepcopy(self.model) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) self.siam_model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) load_lxmert_qa(args.load_lxmert_qa, self.siam_model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() self.siam_model = self.siam_model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() self.siam_model.lxrt_encoder.multi_gpu() # Losses and optimizer self.bce_loss = nn.BCEWithLogitsLoss() self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1) if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(list(self.model.parameters()), args.lr) self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self, env, listener, tok): self.env = env self.feature_size = self.env.feature_size self.tok = tok self.tok.finalize() self.listener = listener # Model self.glove_dim = 300 with open( '/students/u5399302/MatterportData/data/obj_stats2/objectvocab_filtered_dyno4.txt', 'r') as f_ov: self.obj_vocab = [k.strip() for k in f_ov.readlines()] glove_matrix = utils.get_glove_matrix(self.obj_vocab, self.glove_dim) self.objencoder = ObjEncoder(glove_matrix.size(0), glove_matrix.size(1), glove_matrix).cuda() print("VOCAB_SIZE", self.tok.vocab_size()) self.encoder = model.SpeakerEncoder(self.feature_size + args.angle_feat_size, args.rnn_dim, args.dropout, bidirectional=args.bidir).cuda() self.decoder = model.SpeakerDecoder(self.tok.vocab_size(), args.wemb, self.tok.word_to_index['<PAD>'], args.rnn_dim, args.dropout).cuda() self.encoder_optimizer = args.optimizer(self.encoder.parameters(), lr=args.lr) self.decoder_optimizer = args.optimizer(self.decoder.parameters(), lr=args.lr) # Evaluation self.softmax_loss = torch.nn.CrossEntropyLoss( ignore_index=self.tok.word_to_index['<PAD>']) # Will be used in beam search self.nonreduced_softmax_loss = torch.nn.CrossEntropyLoss( ignore_index=self.tok.word_to_index['<PAD>'], size_average=False, reduce=False)
def setup_optimizers(self): if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(list(self.model.parameters()), args.lr)
def __init__(self, dataset): # Built up the model self.tok = dataset.tok self.feature_size = 2048 if args.model == 'init': self.encoder = SpeakerEncoder(self.feature_size).cuda() ctx_size = self.feature_size self.decoder = SpeakerDecoderTran(self.tok.vocab_size, ctx_size, heads=1).cuda() elif args.model == 'heads': self.encoder = MultiCtxEncoder(self.feature_size).cuda() ctx_size = self.encoder.ctx_dim self.decoder = MultiCtxDecoder(self.tok.vocab_size, ctx_size, heads=2).cuda() elif args.model == 'crossatt': self.encoder = CrossAttEncoder(self.feature_size).cuda() ctx_size = self.encoder.ctx_dim self.decoder = MultiCtxDecoder(self.tok.vocab_size, ctx_size, heads=2).cuda() elif args.model == 'newheads': self.encoder = MultiCtxEncoder(self.feature_size).cuda() ctx_size = self.encoder.ctx_dim self.decoder = NewCtxDecoder(self.tok.vocab_size, ctx_size, heads=2).cuda() elif args.model == 'newcross': self.encoder = NewAttEncoder(self.feature_size).cuda() ctx_size = self.encoder.ctx_dim self.decoder = NewCtxDecoder(self.tok.vocab_size, ctx_size, heads=2).cuda() elif args.model == 'dynamic': self.encoder = MultiCtxEncoder(self.feature_size).cuda() ctx_size = self.encoder.ctx_dim self.decoder = DynamicDecoderFC(self.tok.vocab_size, ctx_size).cuda() elif args.model == 'dynamicmh': self.encoder = MultiCtxEncoder(self.feature_size).cuda() ctx_size = self.encoder.ctx_dim self.decoder = DynamicDecoderMH(self.tok.vocab_size, ctx_size).cuda() elif args.model == 'dmc': self.encoder = MultiCtxEncoder(self.feature_size).cuda() ctx_size = self.encoder.ctx_dim self.decoder = DynamicDecoderMHC(self.tok.vocab_size, ctx_size).cuda() if args.baseline == 'linear': self.critic = LinearAct(args.hid_dim, 1).cuda() # Optimizer self.optim = args.optimizer(list(self.encoder.parameters()) + list(self.decoder.parameters()), lr=args.lr) # Logs self.output = args.output os.makedirs(self.output, exist_ok=True) self.writer = SummaryWriter(log_dir=self.output) # Tensorboard summary writer # Loss self.softmax_loss = torch.nn.CrossEntropyLoss(ignore_index=self.tok.pad_id)
def __init__(self,folder="/",load=True): # Datasets if load: self.train_tuple = get_data_tuple( args.train, bs=args.batch_size, shuffle=True, drop_last=True,folder=folder ) if args.valid != "": self.valid_tuple = get_data_tuple( args.valid, bs=128, shuffle=False, drop_last=False, folder=folder,nops=args.nops ) else: self.valid_tuple = None get_bias(self.train_tuple.dataset,self.valid_tuple.dataset) # Model # self.model = VQAModel(self.train_tuple.dataset.num_answers) label2ans = json.load(open("/data/datasets/vqa_mutant/data/vqa/mutant_l2a/mutant_label2ans.json")) self.model = VQAModel(len(label2ans)) self.debias = LearnedMixin(w=0.36,hid_dim=self.model.lxrt_encoder.dim) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() self.debias = self.debias.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if load : if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self): # Datasets self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) if args.valid != "": valid_bsize = args.get("valid_batch_size", 16) self.valid_tuple = get_data_tuple(args.valid, bs=valid_bsize, shuffle=False, drop_last=False) else: self.valid_tuple = None # Model self.model = VQAModel(self.train_tuple.dataset.num_answers) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.get("load_lxmert_pretrain", None) is not None: load_lxmert_from_pretrain_noqa(args.load_lxmert_pretrain, self.model) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() self.model.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self, env, tok, obs_dim, latent_dim, loss_type="mse", view_num=args.view_num, path_len=2): super(BaseVAE, self).__init__() self.env = env self.tok = tok self.obs_dim = obs_dim self.latent_dim = latent_dim self.path_len = path_len self.view_num = view_num self.loss_type = loss_type self.encoder = StateEncoder(obs_dim, latent_dim) self.decoder = StateDecoder(obs_dim, latent_dim, view_num, path_len=path_len) self.policy = PolicyDecoder(obs_dim, latent_dim, view_num, path_len) self.unit_n = Normal(Variable(torch.zeros(1, latent_dim)).cuda(), log_var=Variable(torch.zeros(1, latent_dim)).cuda()) self.encoder_optimizer = args.optimizer(self.encoder.parameters(), lr=args.lr) self.decoder_optimizer = args.optimizer(self.decoder.parameters(), lr=args.lr) self.policy_optimizer = args.optimizer(self.policy.parameters(), lr=args.lr) self.vae_loss_weight = args.vae_loss_weight self.vae_kl_weight = args.vae_kl_weight # self.bc_weight = 100 self.vae_bc_weight = args.vae_bc_weight self.iter = None # training iteration indicator
def __init__(self, env, results_path, tok, episode_len=20): super(Seq2PolicyAgent, self).__init__(env, results_path) self._iter = 0 self.tok = tok self.episode_len = episode_len self.feature_size = self.env.feature_size # Models enc_hidden_size = args.rnn_dim//2 if args.bidir else args.rnn_dim self.encoder = model.EncoderLSTM(tok.vocab_size(), args.wemb, enc_hidden_size, padding_idx, args.dropout, bidirectional=args.bidir).cuda() if args.original_decoder: self.decoder = model.AttnDecoderLSTM(args.aemb, args.rnn_dim, args.dropout, feature_size=self.feature_size + args.angle_feat_size).cuda() else: self.decoder = model.AttnPolicyLSTM(args.aemb, args.rnn_dim, args.dropout, feature_size=self.feature_size + args.angle_feat_size, latent_dim=args.vae_latent_dim).cuda() if args.fix_vae: print("fix the parameters in sub policy") for param in self.decoder.policy.parameters(): param.requires_grad = False self.critic = model.Critic().cuda() self.models = (self.encoder, self.decoder, self.critic) # Optimizers self.encoder_optimizer = args.optimizer(self.encoder.parameters(), lr=args.lr) self.decoder_optimizer = args.optimizer(self.decoder.parameters(), lr=args.lr) self.critic_optimizer = args.optimizer(self.critic.parameters(), lr=args.lr) self.optimizers = (self.encoder_optimizer, self.decoder_optimizer, self.critic_optimizer) # Evaluations self.losses = [] self.criterion = nn.CrossEntropyLoss(ignore_index=args.ignoreid, size_average=False) # Logs sys.stdout.flush() self.logs = defaultdict(list)
def prepare_model(self): # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr)
def __init__(self): self.train_tuple = get_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) if args.valid != "": valid_bsize = 2048 if args.multiGPU else 512 self.valid_tuple = get_tuple(args.valid, bs=valid_bsize, shuffle=False, drop_last=False) else: self.valid_tuple = None self.model = NLVR2Model() # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) # GPU options if args.multiGPU: self.model.lxrt_encoder.multi_gpu() self.model = self.model.cuda() # Losses and optimizer self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1) if "bert" in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(list(self.model.parameters()), args.lr) self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self, folder="/", load=True): # Datasets if load: self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True, folder=folder) if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=512, shuffle=False, drop_last=False, folder=folder) else: self.valid_tuple = None # Model # self.model = VQAModel(self.train_tuple.dataset.num_answers) self.model = VQAModel(len(self.train_tuple.dataset.label2ans), fn_type=args.fn_type) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Load IndexList of Answer to Type Map self.indexlist = json.load( open( "/data/datasets/vqa_mutant/data/vqa/mutant_l2a/mutant_merge_indexlist.json" )) print("Length of Masks", len(self.indexlist), flush=True) indextensor = torch.cuda.LongTensor(self.indexlist) self.mask0 = torch.eq(indextensor, 0).float() self.mask1 = torch.eq(indextensor, 1).float() self.mask2 = torch.eq(indextensor, 2).float() self.mask3 = torch.eq(indextensor, 3).float() self.mask_cache = {} # Loss and Optimizer self.logsoftmax = nn.LogSoftmax() self.sigmoid = nn.Sigmoid() self.softmax = nn.Softmax() self.bceloss = nn.BCELoss() self.nllloss = nn.NLLLoss() self.bce_loss = nn.BCEWithLogitsLoss() self.ce_loss = nn.CrossEntropyLoss() if load: if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self, attention=False): # Datasets print("Fetching data") self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True, dataset_name="test") print("Got data") print("fetching val data") if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=args.batch_size, shuffle=False, drop_last=False, dataset_name="test") print("got data") else: self.valid_tuple = None print("Got data") # Model print("Making model") self.model = VQAModel(self.train_tuple.dataset.num_answers, attention) print("Ready model") # Print model info: print("Num of answers:") print(self.train_tuple.dataset.num_answers) # print("Model info:") # print(self.model) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self, folder="/", load=True): # Datasets if load: self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True, folder=folder) if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=128, shuffle=False, drop_last=False, folder=folder, nops=args.nops) else: self.valid_tuple = None # Model self.model = VQAModel(self.train_tuple.dataset.num_answers) # is_cp=False # if "vqacpv2" in folder: # is_cp=True # if not is_cp: # self.model = VQAModel(3129) # else: # self.model = VQAModel(16039) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() ans_embed = np.load( "/data/datasets/vqa_mutant/data/vqa/mutant_l2a/answer_embs.npy" ) + 1e-8 ans_embed = torch.tensor(ans_embed).cuda() self.ans_embed = torch.nn.functional.normalize(ans_embed, dim=1) self.embed_cache = {} # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if load: if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True) self.cos = nn.CosineSimilarity()
def __init__(self): self.train_type = args.train_type self.device = torch.device(args.device) # Dataloaders for train and val set if not args.test: self.valid_tuple = get_data_tuple(args.valid, bs=args.batch_size, shuffle=False, drop_last=False) self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) num_answers = self.train_tuple.dataset.num_answers file_name = args.train log_str = f"\n{ctime()} || Loaded train set of size {len(self.train_tuple[0])} and val set of size {len(self.valid_tuple[0])}." else: self.test_tuple = get_data_tuple(args.test, bs=args.batch_size, shuffle=False, drop_last=False) num_answers = self.test_tuple.dataset.num_answers file_name = args.test log_str = ( f"\n{ctime()} || Loaded test set of size {len(self.test_tuple[0])}." ) # get dataset name self.dtype = args.task # Model self.model = eUGModel(self.train_type, num_answers, self.dtype, args.model) # Load pre-trained weights if self.train_type == "expl" and args.bb_path is not None: self.model.load_state_dict(torch.load(args.bb_path)) # freeze backbone for p, n in self.model.named_parameters(): if "decoder.model.transformer" not in p: n.requires_grad = False elif args.load_pretrained is not None: self.model.encoder.load(args.load_pretrained) self.model = self.model.to(self.device) # Loss and Optimizer if not args.test: if self.dtype == "vqa_x": self.loss_func = nn.BCEWithLogitsLoss() else: self.loss_func = nn.CrossEntropyLoss() batch_per_epoch = len(self.train_tuple.loader) / args.grad_accum t_total = int(batch_per_epoch * args.epochs) if "bert" in args.optim: print("BertAdam Total Iters: %d" % t_total) from src.optimization import BertAdam self.optim = BertAdam( list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total, ) else: self.optim = args.optimizer(self.model.parameters(), args.lr) self.scheduler = get_linear_schedule_with_warmup( self.optim, num_warmup_steps=args.warmup_steps, num_training_steps=t_total, ) self.grad_accum = args.grad_accum # Output Directory self.output = args.output self.save_steps = args.save_steps os.makedirs(self.output, exist_ok=True) # print logs log_str += f"\n{ctime()} || Model loaded. Batch size {args.batch_size*args.grad_accum} | lr {args.lr} | task: {self.dtype} | type: {self.train_type}." print_log(args, log_str)
def __init__(self): self.train_tuple = get_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) if args.valid != "": valid_bsize = 2048 if args.multiGPU else 512 self.valid_tuple = get_tuple(args.valid, bs=valid_bsize, shuffle=False, drop_last=False) else: self.valid_tuple = None self.model = GQAModel(self.train_tuple.dataset.num_answers) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: self.new_ans_label = load_lxmert_qa( args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Losses and optimizer self.bce_loss = nn.BCEWithLogitsLoss() self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1) # self.KL_loss = nn.KLDivLoss(reduction='none') if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(list(self.model.parameters()), args.lr) self.output = args.output os.makedirs(self.output, exist_ok=True) # Tensorboard self.boards_dir = os.path.join('boards', self.output) if not os.path.exists(self.boards_dir): os.makedirs(self.boards_dir) self.writerTbrd = SummaryWriter(self.boards_dir) # get Glove projection for all answers if args.answer_loss == 'glove': path_glove = './data/GloVe/GloVeDict.pkl' with open(path_glove, 'rb') as f: glove_dic = pickle.load(f) glove_dim = glove_dic['the'].shape[-1] print("Loading Glove%d answer's vector" % glove_dim) self.labelans2glove = [] self.valid_ans_embed = [1] * len( self.train_tuple.dataset.label2ans) for label, ans in enumerate(self.train_tuple.dataset.label2ans): ans = ans.split(' ') glove_ans = [] for w in ans: #print(w) try: glove_ans.append(glove_dic[w]) except KeyError: #print('Full ans: %s' % ans) #input(' ') self.valid_ans_embed[label] = 0 glove_ans.append(np.zeros(glove_dim)) #print(glove_ans) glove_ans = torch.tensor(glove_ans).mean(-2) self.labelans2glove.append(torch.tensor(glove_ans)) #print(self.labelans2glove) print( 'Ratio of valid ans embedding: %f' % (float(sum(self.valid_ans_embed)) / len(self.valid_ans_embed))) self.labelans2glove = torch.stack( self.labelans2glove).float().cuda() self.cosineSim = torch.nn.CosineSimilarity(dim=1, eps=1e-08)
def finetune(self, train_tuple, eval_tuple): # log output_1 = os.path.join(self.output, 'finetune_1') os.makedirs(output_1, exist_ok=True) output_2 = os.path.join(self.output, 'finetune_2') os.makedirs(output_2, exist_ok=True) # Tensorboard boards_dir_1 = os.path.join(self.boards_dir, 'finetune_1') if not os.path.exists(boards_dir_1): os.makedirs(boards_dir_1) boards_dir_2 = os.path.join(self.boards_dir, 'finetune_2') if not os.path.exists(boards_dir_2): os.makedirs(boards_dir_2) # Params lr_1 = args.lr lr_2 = args.lr / 10 epochs_1 = 4 #int(args.epochs / 3) epochs_2 = args.epochs - epochs_1 # Step 0: evaluate pretraining if self.valid_tuple is not None: # Do Validation valid_score = self.evaluate(eval_tuple) print("Before finetune: Valid %0.2f\n" % (valid_score * 100.)) # Step 0.1: finetune new ans only # new_ans_params = [] # for name, p in self.model.named_parameters(): # if "logit_fc.3" in name: # for idx in range(p.size(0)): # if idx in self.new_ans_label: # new_ans_params.append({'params': p[idx]}) # args.epochs = epochs_0 # from lxrt.optimization import BertAdam # self.optim = BertAdam(new_ans_params, # lr=lr_1, # warmup=0.0, # t_total=-1) # print('### Start finetuning new ans...') # self.train(train_tuple, eval_tuple) # First step, only updates answer head #self.optim = torch.optim.Adamax(list(self.model.parameters()), lr_1) #self.optim = torch.optim.SGD(list(self.model.parameters()), lr_1) args.epochs = epochs_1 batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * epochs_1) print("Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam( list(self.model.parameters()), lr=lr_1, warmup=0.0, #!0.034 t_total=-1) # loaded_optim = torch.load("%s_LXRT.pth" % args.load_lxmert_qa)['optimizer'] # self.optim.load_state_dict(loaded_optim) # for group in loaded_optim.param_groups: # for p in group['params']: # if p in loaded_optim['state']: # self.optim.state[p] = loaded_optim.state[p] self.writerTbrd = SummaryWriter(boards_dir_1) self.output = output_1 for name, p in self.model.named_parameters(): if "logit_fc" in name: p.requires_grad = True else: p.requires_grad = False print('### Start finetuning step 1...') self.train(train_tuple, eval_tuple) # Second step, finetune all for name, p in self.model.named_parameters(): p.requires_grad = True if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * epochs_2) print("Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=lr_2, warmup=0.1, t_total=t_total, lr_min=1e-7) else: self.optim = args.optimizer(list(self.model.parameters()), lr_2) args.epochs = epochs_2 self.writerTbrd = SummaryWriter(boards_dir_2) self.output = output_2 print('### Start finetuning step 2...') self.train(train_tuple, eval_tuple)
def __init__(self): # Datasets self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=1024, shuffle=False, drop_last=False) else: self.valid_tuple = None # Model self.model = VQAModel(self.train_tuple.dataset.num_answers, finetune_strategy=args.finetune_strategy) # if finetune strategy is spottune if args.finetune_strategy in PolicyStrategies: self.policy_model = PolicyLXRT( PolicyStrategies[args.finetune_strategy]) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.finetune_strategy in PolicyStrategies: self.policy_model = self.policy_model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() self.policy_model.policy_lxrt_encoder.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Optimizer for policy net if args.finetune_strategy in PolicyStrategies: self.policy_optim = args.policy_optimizer( self.policy_model.parameters(), args.policy_lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)