def __init__(self): # Datasets self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True, dataset_type=args.dataset_type) if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=128, shuffle=False, drop_last=False, dataset_type=args.dataset_type) else: self.valid_tuple = None # Model self.model = VQAModel( self.train_tuple.dataset.num_answers if not args.transfer_learning else VQADataset.get_answers_number(), encoder_type=args.encoder_type) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) self.prepare_model() # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self, folder="/", load=True): # Datasets if load: self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True, folder=folder) if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=128, shuffle=False, drop_last=False, folder=folder, nops=args.nops) else: self.valid_tuple = None # Model # self.model = VQAModel(self.train_tuple.dataset.num_answers) is_cp = False if "vqacpv2" in folder: is_cp = True if not is_cp: self.model = VQAModel(3129) else: self.model = VQAModel(16039) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if load: if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self): # Datasets self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) if args.bert_type == 'ft': bs_infer = 256 else: bs_infer = 1024 if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=bs_infer, shuffle=False, drop_last=False) else: self.valid_tuple = None print("args.lr is {0}".format(args.lr)) # Model self.model = VQAModel(self.train_tuple.dataset.num_answers) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if 'bert' in args.optim: # if type(args.lr) == type("sdfg"): # args.lr = float(args.lr) batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total, schedule=args.lr_schedule, args=args) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self): self.train_tuple = get_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) if args.valid != "": valid_bsize = 2048 if args.multiGPU else 512 self.valid_tuple = get_tuple(args.valid, bs=valid_bsize, shuffle=False, drop_last=False) else: self.valid_tuple = None self.model = GQAModel(self.train_tuple.dataset.num_answers) self.momentum = 0.99997 self.siam_model = copy.deepcopy(self.model) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) self.siam_model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) load_lxmert_qa(args.load_lxmert_qa, self.siam_model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() self.siam_model = self.siam_model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() self.siam_model.lxrt_encoder.multi_gpu() # Losses and optimizer self.bce_loss = nn.BCEWithLogitsLoss() self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1) if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(list(self.model.parameters()), args.lr) self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self): # Datasets self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) if args.valid != "": valid_bsize = args.get("valid_batch_size", 16) self.valid_tuple = get_data_tuple(args.valid, bs=valid_bsize, shuffle=False, drop_last=False) else: self.valid_tuple = None # Model self.model = VQAModel(self.train_tuple.dataset.num_answers) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.get("load_lxmert_pretrain", None) is not None: load_lxmert_from_pretrain_noqa(args.load_lxmert_pretrain, self.model) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() self.model.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self,folder="/",load=True): # Datasets if load: self.train_tuple = get_data_tuple( args.train, bs=args.batch_size, shuffle=True, drop_last=True,folder=folder ) if args.valid != "": self.valid_tuple = get_data_tuple( args.valid, bs=128, shuffle=False, drop_last=False, folder=folder,nops=args.nops ) else: self.valid_tuple = None get_bias(self.train_tuple.dataset,self.valid_tuple.dataset) # Model # self.model = VQAModel(self.train_tuple.dataset.num_answers) label2ans = json.load(open("/data/datasets/vqa_mutant/data/vqa/mutant_l2a/mutant_label2ans.json")) self.model = VQAModel(len(label2ans)) self.debias = LearnedMixin(w=0.36,hid_dim=self.model.lxrt_encoder.dim) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() self.debias = self.debias.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if load : if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def load_model(self): """ Load the pre-trained VQA model """ print(args.n_head) # update data, to allow multiple load_model() calls self.data_loader = Demo_data(self.cfg) # load answer dict with open(self.cfg['answers_dict'], 'r') as f: self.label_to_ans = json.load(f) # load architecture self.model = GQAModel(self.cfg['num_answers']) # load pretrained weights if self.cfg['ecai_lxmert']: if self.cfg['tiny_lxmert']: print( "Sorry, there is no tiny version of ecai lxmert. Change config in src/task/demo_cfg.json!" ) exit(0) # Load raw pretrained model path = self.cfg['pretrained_model_ecai'] _ = load_lxmert_qa(path, self.model, label2ans=self.label_to_ans) else: print(self.cfg) # Load finetuned model if self.cfg['tiny_lxmert']: if self.cfg['oracle']: path = self.cfg['pretrained_model_tiny_lxmert_oracle'] else: path = self.cfg['pretrained_model_tiny_lxmert'] else: if self.cfg['oracle']: print('Oracle model is only available in tiny version!') exit(0) else: path = self.cfg['pretrained_model_lxmert'] print("Load model's weights from %s" % path) state_dict = torch.load("%s.pth" % path, map_location=torch.device('cpu')) for key in list(state_dict.keys()): if '.module' in key: state_dict[key.replace('.module', '')] = state_dict.pop(key) self.model.load_state_dict(state_dict, strict=False) # To GPU # self.model = self.model.cuda() print("Model loaded!")
def __init__(self, folder="/", load=True): # Datasets if load: self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True, folder=folder) if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=512, shuffle=False, drop_last=False, folder=folder) else: self.valid_tuple = None # Model # self.model = VQAModel(self.train_tuple.dataset.num_answers) self.model = VQAModel(len(self.train_tuple.dataset.label2ans), fn_type=args.fn_type) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Load IndexList of Answer to Type Map self.indexlist = json.load( open( "/data/datasets/vqa_mutant/data/vqa/mutant_l2a/mutant_merge_indexlist.json" )) print("Length of Masks", len(self.indexlist), flush=True) indextensor = torch.cuda.LongTensor(self.indexlist) self.mask0 = torch.eq(indextensor, 0).float() self.mask1 = torch.eq(indextensor, 1).float() self.mask2 = torch.eq(indextensor, 2).float() self.mask3 = torch.eq(indextensor, 3).float() self.mask_cache = {} # Loss and Optimizer self.logsoftmax = nn.LogSoftmax() self.sigmoid = nn.Sigmoid() self.softmax = nn.Softmax() self.bceloss = nn.BCELoss() self.nllloss = nn.NLLLoss() self.bce_loss = nn.BCEWithLogitsLoss() self.ce_loss = nn.CrossEntropyLoss() if load: if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self, folder="/", load=True): # Datasets if load: self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True, folder=folder) if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=128, shuffle=False, drop_last=False, folder=folder, nops=args.nops) else: self.valid_tuple = None # Model self.model = VQAModel(self.train_tuple.dataset.num_answers) # is_cp=False # if "vqacpv2" in folder: # is_cp=True # if not is_cp: # self.model = VQAModel(3129) # else: # self.model = VQAModel(16039) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() ans_embed = np.load( "/data/datasets/vqa_mutant/data/vqa/mutant_l2a/answer_embs.npy" ) + 1e-8 ans_embed = torch.tensor(ans_embed).cuda() self.ans_embed = torch.nn.functional.normalize(ans_embed, dim=1) self.embed_cache = {} # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if load: if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True) self.cos = nn.CosineSimilarity()
def __init__(self, args, train_loader=None, val_loader=None, logger=None, num_answers=0, train=True): self.args = args self.max_text_length = args.max_text_length self.train_loader = train_loader self.val_loader = val_loader self.num_answers = num_answers self.logger = logger # Model self.model = GQAModel.from_pretrained("bert-base-uncased", args=args, num_answers=self.num_answers) self.verbose = True if self.args.distributed: if self.args.gpu != 0: self.verbose = False # Load Checkpoint self.start_epoch = None if args.load is not None: path = args.load + '.pth' self.load(path, verbose=self.verbose) elif args.load_lxmert_qa is not None: path = args.load_lxmert_qa + '_LXRT.pth' load_lxmert_qa( args, path, self.model, label2ans=self.train_loader.dataset.raw_dataset.label2ans, verbose=self.verbose) # GPU Options print(f'Model Launching at GPU {self.args.gpu}') from time import time start = time() self.model.cuda(args.gpu) # Optimizer if train: self.optim, self.lr_scheduler = self.create_optimizer_and_scheduler( ) self.bce_loss = nn.BCEWithLogitsLoss() if args.multiGPU: assert args.distributed self.model = DDP(self.model, device_ids=[args.gpu], find_unused_parameters=True) if args.gpu == 0: print(f'It took {time() - start:.1f}s') # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self): self.train_tuple = get_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) if args.valid != "": valid_bsize = 2048 if args.multiGPU else 512 self.valid_tuple = get_tuple(args.valid, bs=valid_bsize, shuffle=False, drop_last=False) else: self.valid_tuple = None self.model = GQAModel(self.train_tuple.dataset.num_answers) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: self.new_ans_label = load_lxmert_qa( args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Losses and optimizer self.bce_loss = nn.BCEWithLogitsLoss() self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1) # self.KL_loss = nn.KLDivLoss(reduction='none') if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(list(self.model.parameters()), args.lr) self.output = args.output os.makedirs(self.output, exist_ok=True) # Tensorboard self.boards_dir = os.path.join('boards', self.output) if not os.path.exists(self.boards_dir): os.makedirs(self.boards_dir) self.writerTbrd = SummaryWriter(self.boards_dir) # get Glove projection for all answers if args.answer_loss == 'glove': path_glove = './data/GloVe/GloVeDict.pkl' with open(path_glove, 'rb') as f: glove_dic = pickle.load(f) glove_dim = glove_dic['the'].shape[-1] print("Loading Glove%d answer's vector" % glove_dim) self.labelans2glove = [] self.valid_ans_embed = [1] * len( self.train_tuple.dataset.label2ans) for label, ans in enumerate(self.train_tuple.dataset.label2ans): ans = ans.split(' ') glove_ans = [] for w in ans: #print(w) try: glove_ans.append(glove_dic[w]) except KeyError: #print('Full ans: %s' % ans) #input(' ') self.valid_ans_embed[label] = 0 glove_ans.append(np.zeros(glove_dim)) #print(glove_ans) glove_ans = torch.tensor(glove_ans).mean(-2) self.labelans2glove.append(torch.tensor(glove_ans)) #print(self.labelans2glove) print( 'Ratio of valid ans embedding: %f' % (float(sum(self.valid_ans_embed)) / len(self.valid_ans_embed))) self.labelans2glove = torch.stack( self.labelans2glove).float().cuda() self.cosineSim = torch.nn.CosineSimilarity(dim=1, eps=1e-08)
def __init__(self): # Datasets self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True) if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=1024, shuffle=False, drop_last=False) else: self.valid_tuple = None # Model self.model = VQAModel(self.train_tuple.dataset.num_answers, finetune_strategy=args.finetune_strategy) # if finetune strategy is spottune if args.finetune_strategy in PolicyStrategies: self.policy_model = PolicyLXRT( PolicyStrategies[args.finetune_strategy]) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.finetune_strategy in PolicyStrategies: self.policy_model = self.policy_model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() self.policy_model.policy_lxrt_encoder.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Optimizer for policy net if args.finetune_strategy in PolicyStrategies: self.policy_optim = args.policy_optimizer( self.policy_model.parameters(), args.policy_lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)
def __init__(self, attention=False): # Datasets print("Fetching data") self.train_tuple = get_data_tuple(args.train, bs=args.batch_size, shuffle=True, drop_last=True, dataset_name="test") print("Got data") print("fetching val data") if args.valid != "": self.valid_tuple = get_data_tuple(args.valid, bs=args.batch_size, shuffle=False, drop_last=False, dataset_name="test") print("got data") else: self.valid_tuple = None print("Got data") # Model print("Making model") self.model = VQAModel(self.train_tuple.dataset.num_answers, attention) print("Ready model") # Print model info: print("Num of answers:") print(self.train_tuple.dataset.num_answers) # print("Model info:") # print(self.model) # Load pre-trained weights if args.load_lxmert is not None: self.model.lxrt_encoder.load(args.load_lxmert) if args.load_lxmert_qa is not None: load_lxmert_qa(args.load_lxmert_qa, self.model, label2ans=self.train_tuple.dataset.label2ans) # GPU options self.model = self.model.cuda() if args.multiGPU: self.model.lxrt_encoder.multi_gpu() # Loss and Optimizer self.bce_loss = nn.BCEWithLogitsLoss() if 'bert' in args.optim: batch_per_epoch = len(self.train_tuple.loader) t_total = int(batch_per_epoch * args.epochs) print("BertAdam Total Iters: %d" % t_total) from lxrt.optimization import BertAdam self.optim = BertAdam(list(self.model.parameters()), lr=args.lr, warmup=0.1, t_total=t_total) else: self.optim = args.optimizer(self.model.parameters(), args.lr) # Output Directory self.output = args.output os.makedirs(self.output, exist_ok=True)