def get_model_vqa(vqa_model="minhmul_noatt_train_2048"): path = "options/breast/{}.yaml".format(vqa_model) args = parser.parse_args() options = { 'vqa': { 'trainsplit': args.vqa_trainsplit }, 'logs': { 'dir_logs': args.dir_logs }, 'model': { 'arch': args.arch, 'seq2vec': { 'type': args.st_type, 'dropout': args.st_dropout, 'fixed_emb': args.st_fixed_emb } }, 'optim': { 'lr': args.learning_rate, 'batch_size': args.batch_size, 'epochs': args.epochs } } with open(path, 'r') as handle: options_yaml = yaml.load(handle) options = vqa_utils.update_values(options, options_yaml) if 'vgenome' not in options: options['vgenome'] = None trainset = datasets.factory_VQA(options['vqa']['trainsplit'], options['vqa'], options['coco'], options['vgenome']) model = models_vqa.factory(options['model'], trainset.vocab_words(), trainset.vocab_answers(), cuda=False, data_parallel=False) # load checkpoint path_ckpt_model = "logs/breast/{}/best_model.pth.tar".format(vqa_model) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") if os.path.isfile(path_ckpt_model): model_state = torch.load(path_ckpt_model, map_location=device) model.load_state_dict(model_state) return model
def main(): global args, options, model, cnn, transform, trainset args = parser.parse_args() options = {'logs': {'dir_logs': args.dir_logs}} if args.path_opt is not None: with open(args.path_opt, 'r') as handle: options_yaml = yaml.load(handle) options = utils.update_values(options, options_yaml) print('## args') pprint(vars(args)) print('## options') pprint(options) trainset = datasets.factory_VQA(options['vqa']['trainsplit'], options['vqa']) #options['coco']) normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) transform = transforms.Compose([ transforms.Scale(options['coco']['size']), transforms.CenterCrop(options['coco']['size']), transforms.ToTensor(), normalize, ]) opt_factory_cnn = {'arch': options['coco']['arch']} cnn = convnets.factory(opt_factory_cnn, cuda=args.cuda, data_parallel=False) model = models.factory(options['model'], trainset.vocab_words(), trainset.vocab_answers(), cuda=args.cuda, data_parallel=False) model.eval() start_epoch, best_acc1, _ = load_checkpoint( model, None, os.path.join(options['logs']['dir_logs'], args.resume)) my_local_ip = '192.168.0.32' my_local_port = 3456 run_simple(my_local_ip, my_local_port, application)
def load_vqa_model(args, dataset, vqa_model="minhmul_noatt_train_2048"): path = "options/{}/{}.yaml".format(dataset, vqa_model) args = parser.parse_args() options = { 'vqa': { 'trainsplit': args.vqa_trainsplit }, 'logs': { 'dir_logs': args.dir_logs }, 'model': { 'arch': args.arch, 'seq2vec': { 'type': args.st_type, 'dropout': args.st_dropout, 'fixed_emb': args.st_fixed_emb } } } with open(path, 'r') as handle: options_yaml = yaml.load(handle) options = utils.update_values(options, options_yaml) if 'vgenome' not in options: options['vgenome'] = None trainset = datasets.factory_VQA(options['vqa']['trainsplit'], options['vqa'], options['coco'], options['vgenome']) model = models_vqa.factory(options['model'], trainset.vocab_words(), trainset.vocab_answers(), cuda=False, data_parallel=False) # load checkpoint path_ckpt_model = "logs/{}/{}/best_model.pth.tar".format( dataset, vqa_model) if os.path.isfile(path_ckpt_model): model = load_dict_torch_031(model, path_ckpt_model) return model
def main(): global args, best_acc1 args = parser.parse_args() ######################################################################################### # Create options ######################################################################################### if args.bert_model == "bert-base-uncased": question_features_path = BASE_EXTRACTED_QUES_FEATURES_PATH elif args.bert_model == "bert-base-multilingual-cased": question_features_path = CASED_EXTRACTED_QUES_FEATURES_PATH else: question_features_path = EXTRACTED_QUES_FEATURES_PATH options = { 'vqa': { 'trainsplit': args.vqa_trainsplit }, 'logs': { 'dir_logs': args.dir_logs }, 'model': { 'arch': args.arch, 'seq2vec': { 'type': args.st_type, 'dropout': args.st_dropout, 'fixed_emb': args.st_fixed_emb } }, 'optim': { 'lr': args.learning_rate, 'batch_size': args.batch_size, 'epochs': args.epochs } } if args.path_opt is not None: with open(args.path_opt, 'r') as handle: options_yaml = yaml.load(handle, Loader=yaml.FullLoader) options = utils.update_values(options, options_yaml) print('## args') pprint(vars(args)) print('## options') pprint(options) if args.help_opt: return # Set datasets options if 'vgenome' not in options: options['vgenome'] = None ######################################################################################### # Create needed datasets ######################################################################################### trainset = datasets.factory_VQA(options['vqa']['trainsplit'], options['vqa'], options['coco'], options['vgenome']) train_loader = trainset.data_loader( batch_size=options['optim']['batch_size'], num_workers=args.workers, shuffle=True) if options['vqa']['trainsplit'] == 'train': valset = datasets.factory_VQA('val', options['vqa'], options['coco']) val_loader = valset.data_loader( batch_size=options['optim']['batch_size'], num_workers=args.workers) if options['vqa']['trainsplit'] == 'trainval' or args.evaluate: testset = datasets.factory_VQA('test', options['vqa'], options['coco']) test_loader = testset.data_loader( batch_size=options['optim']['batch_size'], num_workers=args.workers) ######################################################################################### # Create model, criterion and optimizer ######################################################################################### model = models.factory(options['model'], trainset.vocab_words(), trainset.vocab_answers(), cuda=True, data_parallel=True) criterion = criterions.factory(options['vqa'], cuda=True) optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, model.parameters()), options['optim']['lr']) ######################################################################################### # args.resume: resume from a checkpoint OR create logs directory ######################################################################################### exp_logger = None if args.resume: args.start_epoch, best_acc1, exp_logger = load_checkpoint( model.module, optimizer, os.path.join(options['logs']['dir_logs'], args.resume)) else: # Or create logs directory if os.path.isdir(options['logs']['dir_logs']): if click.confirm( 'Logs directory already exists in {}. Erase?'.format( options['logs']['dir_logs'], default=False)): os.system('rm -r ' + options['logs']['dir_logs']) else: return os.system('mkdir -p ' + options['logs']['dir_logs']) path_new_opt = os.path.join(options['logs']['dir_logs'], os.path.basename(args.path_opt)) path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml') with open(path_new_opt, 'w') as f: yaml.dump(options, f, default_flow_style=False) with open(path_args, 'w') as f: yaml.dump(vars(args), f, default_flow_style=False) if exp_logger is None: # Set loggers exp_name = os.path.basename( options['logs']['dir_logs']) # add timestamp exp_logger = logger.Experiment(exp_name, options) exp_logger.add_meters('train', make_meters()) exp_logger.add_meters('test', make_meters()) if options['vqa']['trainsplit'] == 'train': exp_logger.add_meters('val', make_meters()) exp_logger.info['model_params'] = utils.params_count(model) print('Model has {} parameters'.format( exp_logger.info['model_params'])) ######################################################################################### # args.evaluate: on valset OR/AND on testset ######################################################################################### if args.evaluate: path_logger_json = os.path.join(options['logs']['dir_logs'], 'logger.json') if options['vqa']['trainsplit'] == 'train': acc1, val_results = engine.validate(val_loader, model, criterion, exp_logger, args.start_epoch, args.print_freq) # save results and compute OpenEnd accuracy exp_logger.to_json(path_logger_json) save_results(val_results, args.start_epoch, valset.split_name(), options['logs']['dir_logs'], options['vqa']['dir']) test_results, testdev_results = engine.test(test_loader, model, exp_logger, args.start_epoch, args.print_freq) # save results and DOES NOT compute OpenEnd accuracy exp_logger.to_json(path_logger_json) save_results(test_results, args.start_epoch, testset.split_name(), options['logs']['dir_logs'], options['vqa']['dir']) save_results(testdev_results, args.start_epoch, testset.split_name(testdev=True), options['logs']['dir_logs'], options['vqa']['dir']) return ######################################################################################### # Begin training on train/val or trainval/test ######################################################################################### for epoch in range(args.start_epoch + 1, options['optim']['epochs']): # if epoch > 1 and gen_utils.str2bool(args.is_augment_image) and 'options/med/' in args.path_opt: # cmd = "python main/extract.py --dir_data data/raw/vqa_med/preprocessed --dataset med --is_augment_image 1 -b 64" # os.system(cmd) # if epoch == 1 and 'options/med/' in args.path_opt: # cmd = "python main/extract.py --dir_data data/raw/vqa_med/preprocessed --dataset med --is_augment_image 0 -b 64" # os.system(cmd) # train for one epoch engine.train(train_loader, model, criterion, optimizer, exp_logger, epoch, args.print_freq, dict=io_utils.read_pickle(question_features_path), bert_dim=options["model"]["dim_q"]) if options['vqa']['trainsplit'] == 'train': # evaluate on validation set acc1, val_results = engine.validate( val_loader, model, criterion, exp_logger, epoch, args.print_freq, topk=5, dict=io_utils.read_pickle(question_features_path), bert_dim=options["model"]["dim_q"]) # remember best prec@1 and save checkpoint is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) save_checkpoint( { 'epoch': epoch, 'arch': options['model']['arch'], 'best_acc1': best_acc1, 'exp_logger': exp_logger }, model.module.state_dict(), optimizer.state_dict(), options['logs']['dir_logs'], args.save_model, args.save_all_from, is_best) # save results and compute OpenEnd accuracy save_results(val_results, epoch, valset.split_name(), options['logs']['dir_logs'], options['vqa']['dir']) else: test_results, testdev_results = engine.test( test_loader, model, exp_logger, epoch, args.print_freq, topk=5, dict=io_utils.read_pickle(question_features_path), bert_dim=options["model"]["dim_q"]) # save checkpoint at every timestep save_checkpoint( { 'epoch': epoch, 'arch': options['model']['arch'], 'best_acc1': best_acc1, 'exp_logger': exp_logger }, model.module.state_dict(), optimizer.state_dict(), options['logs']['dir_logs'], args.save_model, args.save_all_from) # save results and DOES NOT compute OpenEnd accuracy save_results(test_results, epoch, testset.split_name(), options['logs']['dir_logs'], options['vqa']['dir']) save_results(testdev_results, epoch, testset.split_name(testdev=True), options['logs']['dir_logs'], options['vqa']['dir'])
def main(): global args, best_acc1 args = parser.parse_args() ######################################################################################### # Create options ######################################################################################### options = { 'vqa' : { 'trainsplit': args.vqa_trainsplit }, 'logs': { 'dir_logs': args.dir_logs }, 'model': { 'arch': args.arch, 'seq2vec': { 'type': args.st_type, 'dropout': args.st_dropout, 'fixed_emb': args.st_fixed_emb } }, 'optim': { 'lr': args.learning_rate, 'batch_size': args.batch_size, 'epochs': args.epochs } } if args.path_opt is not None: with open(args.path_opt, 'r') as handle: options_yaml = yaml.load(handle) options = utils.update_values(options, options_yaml) # print('## args'); pprint(vars(args)) # print('## options'); pprint(options) if args.help_opt: return # Set datasets options if 'vgenome' not in options: options['vgenome'] = None ######################################################################################### # Create needed datasets ######################################################################################### trainset = datasets.factory_VQA(options['vqa']['trainsplit'], options['vqa'], options['coco'], options['vgenome']) train_loader = trainset.data_loader(batch_size=options['optim']['batch_size'], num_workers=args.workers, shuffle=True) imgs_trainset = CustomDataset(root_dir='FLIR/', transform=None) imgs_train_loader = DataLoader(imgs_trainset, batch_size=4) # if options['vqa']['trainsplit'] == 'train': # valset = datasets.factory_VQA('val', options['vqa'], options['coco']) # val_loader = valset.data_loader(batch_size=options['optim']['batch_size'], # num_workers=args.workers) if options['vqa']['trainsplit'] == 'trainval' or args.evaluate: testset = datasets.factory_VQA('test', options['vqa'], options['coco']) test_loader = testset.data_loader(batch_size=options['optim']['batch_size'], num_workers=args.workers) ######################################################################################### # Create model, criterion and optimizer ######################################################################################### model = models.factory(options['model'], trainset.vocab_words(), trainset.vocab_answers(), cuda=True, data_parallel=True) criterion = criterions.factory(options['vqa'], cuda=True) optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), options['optim']['lr']) ######################################################################################### # args.resume: resume from a checkpoint OR create logs directory ######################################################################################### exp_logger = None if args.resume: args.start_epoch, best_acc1, exp_logger = load_checkpoint(model.module, optimizer, os.path.join(options['logs']['dir_logs'], args.resume)) # else: # # Or create logs directory # if os.path.isdir(options['logs']['dir_logs']): # if click.confirm('Logs directory already exists in {}. Erase?' # .format(options['logs']['dir_logs'], default=False)): # os.system('rm -r ' + options['logs']['dir_logs']) # else: # return # os.system('mkdir -p ' + options['logs']['dir_logs']) # path_new_opt = os.path.join(options['logs']['dir_logs'], # os.path.basename(args.path_opt)) # path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml') # with open(path_new_opt, 'w') as f: # yaml.dump(options, f, default_flow_style=False) # with open(path_args, 'w') as f: # yaml.dump(vars(args), f, default_flow_style=False) if exp_logger is None: # Set loggers exp_name = os.path.basename(options['logs']['dir_logs']) # add timestamp exp_logger = logger.Experiment(exp_name, options) exp_logger.add_meters('train', make_meters()) exp_logger.add_meters('test', make_meters()) if options['vqa']['trainsplit'] == 'train': exp_logger.add_meters('val', make_meters()) exp_logger.info['model_params'] = utils.params_count(model) print('Model has {} parameters'.format(exp_logger.info['model_params'])) ######################################################################################### # args.evaluate: on valset OR/AND on testset ######################################################################################### # if args.evaluate: # path_logger_json = os.path.join(options['logs']['dir_logs'], 'logger.json') # if options['vqa']['trainsplit'] == 'train': # acc1, val_results = engine.validate(val_loader, model, criterion, # exp_logger, args.start_epoch, args.print_freq) # # save results and compute OpenEnd accuracy # exp_logger.to_json(path_logger_json) # save_results(val_results, args.start_epoch, valset.split_name(), # options['logs']['dir_logs'], options['vqa']['dir']) # test_results, testdev_results = engine.test(test_loader, model, exp_logger, # args.start_epoch, args.print_freq) # # save results and DOES NOT compute OpenEnd accuracy # exp_logger.to_json(path_logger_json) # save_results(test_results, args.start_epoch, testset.split_name(), # options['logs']['dir_logs'], options['vqa']['dir']) # save_results(testdev_results, args.start_epoch, testset.split_name(testdev=True), # options['logs']['dir_logs'], options['vqa']['dir']) # return ######################################################################################### # Begin training on train/val or trainval/test ######################################################################################### print("Started training") for epoch in range(args.start_epoch+1, options['optim']['epochs']): #adjust_learning_rate(optimizer, epoch) # train for one epoch engine.train(imgs_train_loader, model, criterion, optimizer, exp_logger, epoch, args.print_freq)
def main(): args = parser.parse_args() ######################################################################################### # Create options ######################################################################################### options = { 'optim': { 'lr': args.learning_rate, 'batch_size': args.batch_size, 'epochs': args.epochs } } with open(args.path_opt, 'r') as handle: options_yaml = yaml.load(handle) options = utils.update_values(options, options_yaml) options['vgenome'] = None ######################################################################################### # Bookkeeping ######################################################################################### if args.resume: run_name = args.resume save_dir = os.path.join('logs', 'cx', run_name) assert(os.path.isdir(save_dir)) i = 1 log_dir = os.path.join('runs', run_name, 'resume_{}'.format(i)) while(os.path.isdir(log_dir)): i += 1 log_dir = os.path.join('runs', run_name, 'resume_{}'.format(i)) else: run_name = datetime.now().strftime('%b%d-%H-%M-%S') if args.comment: run_name += '_' + args.comment save_dir = os.path.join('logs', 'cx', run_name) if os.path.isdir(save_dir): if click.confirm('Save directory already exists in {}. Erase?'.format(save_dir)): os.system('rm -r ' + save_dir) else: return os.makedirs(os.path.join(save_dir, 'ckpt')) os.makedirs(os.path.join(save_dir, 'best')) # Tensorboard log directory log_dir = os.path.join('runs', run_name) train_writer = SummaryWriter(log_dir=os.path.join(log_dir, 'train')) val_writer = SummaryWriter(log_dir=os.path.join(log_dir, 'val')) print('Saving model to {}'.format(save_dir)) print('Logging results to {}'.format(log_dir)) ######################################################################################### # Create datasets ######################################################################################### print('=> Loading VQA dataset...') if args.dev_mode: trainset_fname = 'trainset_augmented_small.pickle' else: trainset_fname = 'trainset_augmented.pickle' trainset = pickle.load(open(os.path.join(options['vqa']['path_trainset'], 'pickle_old', trainset_fname), 'rb')) # if not args.dev_mode: valset_fname = 'valset_augmented_small.pickle' valset = pickle.load(open(os.path.join(options['vqa']['path_trainset'], 'pickle_old', valset_fname), 'rb')) print('=> Loading KNN data...') knns = json.load(open(options['coco']['path_knn'], 'r')) knns = {int(k):v for k,v in knns.items()} print('=> Loading COCO image features...') features_train = h5py.File(os.path.join(options['coco']['path_raw'], 'trainset.hdf5'), 'r').get('noatt') features_train = np.array(features_train) # if not args.dev_mode: features_val = h5py.File(os.path.join(options['coco']['path_raw'], 'valset.hdf5'), 'r').get('noatt') features_val = np.array(features_val) ######################################################################################### # Create model ######################################################################################### print('=> Building model...') vqa_model = models.factory(options['model'], trainset['vocab_words'], trainset['vocab_answers'], cuda=True, data_parallel=True) vqa_model = vqa_model.module if args.pretrained_vqa: load_vqa_checkpoint(vqa_model, None, os.path.join(options['logs']['dir_logs'], 'best')) cx_model = ContrastiveModel(vqa_model, knn_size=2, trainable_vqa=args.trainable_vqa) optimizer = torch.optim.Adam(cx_model.parameters(), lr=options['optim']['lr']) if args.resume: info, start_epoch, best_recall = load_cx_checkpoint(cx_model, save_dir, resume_best=args.best) else: info = [] start_epoch = 1 best_recall = 0 cx_model.cuda() ######################################################################################### # Train loop ######################################################################################### print('=> Starting training...') if args.pairwise: print('==> Pairwise training') for epoch in range(start_epoch, options['optim']['epochs'] + 1): cx_model.train() if vqa_model is not None: if args.trainable_vqa: vqa_model.train() else: vqa_model.eval() train_b = 0 criterion = ContrastiveLoss() criterion.cuda() trainset_batched = batchify(trainset['examples_list'], batch_size=options['optim']['batch_size']) for batch in tqdm(trainset_batched): assert(cx_model.training) if vqa_model is not None: if args.trainable_vqa: assert(vqa_model.training) else: assert(not vqa_model.training) image_features, question_wids, answer_aids, comp_idxs = getDataFromBatch(batch, features_train, trainset['name_to_index'], pairwise=args.pairwise) h_out = cx_model(image_features, question_wids, answer_aids) loss_comp = criterion(h_out[:, 0], h_out[:, 1], label=Variable(torch.ones([len(batch)])).cuda()) loss_other = criterion(h_out[:, 0], h_out[:, 2], label=Variable(torch.zeros([len(batch)])).cuda()) loss = loss_comp + loss_other if optimizer is not None: optimizer.zero_grad() loss.backward() optimizer.step() train_b += 1 if train_b % args.print_freq == 0: dists = cx_model.get_scores(h_out[:, 0], h_out[:, 1:]) dist_comp = dists[:, 0].mean() dist_other = dists[:, 1].mean() metrics = { 'contrastive/loss_comp': float(loss_comp), 'contrastive/loss_other': float(loss_other), 'contrastive/loss': float(loss), 'contrastive/dist_comp': float(dist_comp), 'contrastive/dist_other': float(dist_other) } log_results(train_writer, mode='train', epoch=epoch, i=((epoch - 1) * len(trainset_batched)) + train_b, metrics=metrics) if (args.eval_freq > 0 and train_b % args.eval_freq == 0) or train_b == len(trainset_batched): eval_results = eval_model(cx_model, valset, features_val, options['optim']['batch_size'], pairwise=args.pairwise) log_results(val_writer, mode='val', epoch=epoch, i=((epoch - 1) * len(trainset_batched)) + train_b, metrics=eval_results) info.append(eval_results) if eval_results['contrastive/recall'] > best_recall: is_best = True best_recall = eval_results['contrastive/recall'] else: is_best = False save_cx_checkpoint(cx_model, info, save_dir, is_best=is_best) eval_results = eval_model(cx_model, valset, features_val, options['optim']['batch_size'], pairwise=args.pairwise) log_results(val_writer, mode='test', epoch=0, i=0, metrics=eval_results)
def main(): args = parser.parse_args() ########################################################################## # Create options ########################################################################## # Parse options options = { 'optim': { 'lr': args.learning_rate, 'batch_size': args.batch_size, 'epochs': args.epochs, }, 'cx_model': { 'pretrained_vqa': args.pretrained_vqa, 'trainable_vqa': args.trainable_vqa, } } with open(args.path_opt, 'r') as handle: options_yaml = yaml.load(handle) options = utils.update_values(options, options_yaml) options['vgenome'] = None # Ensure determinism random.seed(42) torch.manual_seed(42) torch.cuda.manual_seed_all(42) # Bookkeeping ########################################################################## if args.cx_model == 'NeuralModel' and not args.comment: args.comment = options['cx_model']['name'] if args.resume: print('hello') run_name = args.resume save_dir = os.path.join(args.project_dir, 'logs', 'cx', run_name) assert (os.path.isdir(save_dir)) i = 1 log_dir = os.path.join(args.project_dir, 'runs', run_name, 'resume_{}'.format(i)) while (os.path.isdir(log_dir)): i += 1 log_dir = os.path.join(args.project_dir, 'runs', run_name, 'resume_{}'.format(i)) else: print('why am i here') run_name = datetime.now().strftime('%b%d-%H-%M-%S') if args.comment: run_name += '_' + args.comment save_dir = os.path.join(args.project_dir, 'logs', 'cx', run_name) if os.path.isdir(save_dir): if click.confirm( 'Save directory already exists in {}. Erase?'.format( save_dir)): os.system('rm -r ' + save_dir) else: return os.makedirs(os.path.join(save_dir, 'ckpt')) os.makedirs(os.path.join(save_dir, 'best')) # Tensorboard log directory log_dir = os.path.join(args.project_dir, 'runs', run_name) if args.viz: viz_dir = os.path.join(args.project_dir, 'viz', 'cx', run_name) print('viz_dir', viz_dir, run_name) if os.path.isdir(viz_dir): if click.confirm( 'Viz directory already exists in {}. Erase?'.format( viz_dir)): os.system('rm -r ' + viz_dir) else: return os.makedirs(viz_dir) train_writer = SummaryWriter(log_dir=os.path.join(log_dir, 'train')) val_writer = SummaryWriter(log_dir=os.path.join(log_dir, 'val')) print('Saving model to {}'.format(save_dir)) print('Logging results to {}'.format(log_dir)) ########################################################################## # Create datasets ########################################################################## print('=> Loading VQA dataset...') if args.dev_mode: trainset_fname = 'trainset_augmented_small.pickle' else: trainset_fname = 'trainset_augmented.pickle' trainset = pickle.load( open( os.path.join(options['vqa']['path_trainset'], 'pickle_old', trainset_fname), 'rb')) valset_fname = 'valset_augmented_small.pickle' valset = pickle.load( open( os.path.join(options['vqa']['path_trainset'], 'pickle_old', valset_fname), 'rb')) if args.test: testset_fname = 'valset_augmented.pickle' testset = pickle.load( open( os.path.join(options['vqa']['path_trainset'], 'pickle_old', testset_fname), 'rb')) print('=> Loading KNN data...') knns = json.load(open(options['coco']['path_knn'], 'r')) knns = {int(k): v for k, v in knns.items()} print('=> Loading COCO image features...') features_train = h5py.File( os.path.join(options['coco']['path_features'], 'trainset.hdf5'), 'r').get('noatt') features_train = np.array(features_train) # if not args.dev_mode: features_val = h5py.File( os.path.join(options['coco']['path_features'], 'valset.hdf5'), 'r').get('noatt') features_val = np.array(features_val) ########################################################################## # Create model ########################################################################## print('=> Building model...') vqa_model = None optimizer = None if args.cx_model == "RandomBaseline": cx_model = RandomBaseline(knn_size=24) elif args.cx_model == "DistanceBaseline": cx_model = DistanceBaseline(knn_size=24) else: vqa_model = models.factory(options['model'], trainset['vocab_words'], trainset['vocab_answers'], cuda=True, data_parallel=True) vqa_model = vqa_model.module if args.pretrained_vqa: load_vqa_checkpoint( vqa_model, None, os.path.join(options['logs']['dir_logs'], 'best')) if args.cx_model == "BlackBox": cx_model = BlackBox(vqa_model, knn_size=24, trainable_vqa=args.trainable_vqa) elif args.cx_model == "LinearContext": cx_model = LinearContext(vqa_model, knn_size=24, trainable_vqa=args.trainable_vqa) elif args.cx_model == "SemanticBaseline": if args.sb_lambda is None: raise ValueError( "If semantic baseline is selected then --sb_lambda must also be provided." ) cx_model = SemanticBaseline(vqa_model, knn_size=24, trainable_vqa=args.trainable_vqa) cx_model.set_lambda(args.sb_lambda) emb = pickle.load( open( os.path.join(options['vqa']['path_trainset'], "answer_embedding.pickle"), 'rb')) cx_model.set_answer_embedding(emb) elif args.cx_model == "NeuralModel": model_spec = options['cx_model'] if model_spec['pretrained_emb']: emb = pickle.load( open( os.path.join(options['vqa']['path_trainset'], "answer_embedding.pickle"), 'rb')) else: emb = None cx_model = NeuralModel(model_spec=model_spec, dim_h=model_spec['dim_h'], n_layers=model_spec['n_layers'], emb=emb, drop_p=model_spec['drop_p'], vqa_model=vqa_model, knn_size=24, trainable_vqa=model_spec['trainable_vqa']) elif args.cx_model == "PairwiseModel": assert (args.pairwise) cx_model = PairwiseModel(vqa_model, knn_size=2, trainable_vqa=args.trainable_vqa) elif args.cx_model == "PairwiseLinearModel": cx_model = PairwiseLinearModel(vqa_model, knn_size=24, trainable_vqa=args.trainable_vqa) else: raise ValueError("Unrecognized cx_model {}".format(args.cx_model)) optimizer = torch.optim.Adam(cx_model.parameters(), lr=options['optim']['lr']) print("Built {}".format(args.cx_model)) if args.resume: info, start_epoch, best_recall = load_cx_checkpoint( cx_model, save_dir, resume_best=args.best) else: info = [] start_epoch = 1 best_recall = 0 cx_model.cuda() ########################################################################## # Train loop ########################################################################## print('=> Starting training...') if args.pairwise: print('==> Pairwise training') epoch = None for epoch in range(start_epoch, options['optim']['epochs'] + 1): cx_model.train() if vqa_model is not None: if options['cx_model']['trainable_vqa']: vqa_model.train() else: vqa_model.eval() train_b = 0 criterion = nn.CrossEntropyLoss(size_average=False) trainset_batched = batchify(trainset['examples_list'], batch_size=options['optim']['batch_size']) for batch in tqdm(trainset_batched): assert (cx_model.training) if vqa_model is not None: if options['cx_model']['trainable_vqa']: assert (vqa_model.training) else: assert (not vqa_model.training) image_features, question_wids, answer_aids, comp_idxs = getDataFromBatch( batch, features_train, trainset['name_to_index'], pairwise=args.pairwise) scores = cx_model(image_features, question_wids, answer_aids) if args.pairwise: assert (scores.size(1) == 2) zeros = Variable(torch.LongTensor([0] * len(batch))).cuda() loss = criterion(scores, zeros) / len(batch) correct = recallAtK(scores, zeros, k=1) else: correct = recallAtK(scores, comp_idxs, k=5) loss = criterion(scores, comp_idxs) / len(batch) if optimizer is not None: optimizer.zero_grad() loss.backward() optimizer.step() train_b += 1 if train_b % args.print_freq == 0: if args.pairwise: metrics = { 'loss_pairwise': float(loss), 'acc_pairwise': (correct.sum() / len(batch)) } else: metrics = { 'loss': float(loss), 'recall': (correct.sum() / len(batch)) } log_results(train_writer, mode='train', epoch=epoch, i=((epoch - 1) * len(trainset_batched)) + train_b, metrics=metrics) if (args.eval_freq > 0 and train_b % args.eval_freq == 0) or train_b == len(trainset_batched): eval_results = eval_model(cx_model, valset, features_val, options['optim']['batch_size'], pairwise=args.pairwise) log_results(val_writer, mode='val', epoch=epoch, i=((epoch - 1) * len(trainset_batched)) + train_b, metrics=eval_results) info.append(eval_results) if info[-1]['recall'] > best_recall: is_best = True best_recall = info[-1]['recall'] else: is_best = False save_cx_checkpoint(cx_model, info, save_dir, is_best=is_best) if args.test or args.viz: if epoch is not None: _, best_epoch, _ = load_cx_checkpoint(cx_model, save_dir, resume_best=True) else: best_epoch = 0 if args.test: test_results = eval_model(cx_model, testset, features_val, options['optim']['batch_size'], pairwise=args.pairwise) test_results['best_epoch'] = best_epoch with open(os.path.join(save_dir, 'final_results.txt'), 'w') as outfile: outstr = json.dumps(test_results) outfile.write(outstr) print('Saved results to {}'.format(save_dir)) print('FINAL RESULTS ON BEST EPOCH {}'.format(best_epoch), test_results) if args.viz: visualize_results(cx_model, valset, features_val, 200, options['coco']['path_val_raw'], viz_dir)
def compute_prob_one_model(model_name, vqa_trainsplit="train"): parser = argparse.ArgumentParser( description='Train/Evaluate models', formatter_class=argparse.ArgumentDefaultsHelpFormatter) ################################################## # yaml options file contains all default choices # # parser.add_argument('--path_opt', default='options/breast/default.yaml', type=str, # help='path to a yaml options file') parser.add_argument( '--path_opt', default='options/med/bilinear_att_train_imagenet_h200_g4.yaml', type=str, help='path to a yaml options file') ################################################ # change cli options to modify default choices # # logs options parser.add_argument( '--dir_logs', default='logs/med/train/globalbilinear_att_train_imagenet_h200_g4', type=str, help='dir logs') # data options parser.add_argument('--vqa_trainsplit', type=str, choices=['train', 'trainval'], default=vqa_trainsplit) # model options parser.add_argument('--arch', choices=models.model_names, help='vqa model architecture: ' + ' | '.join(models.model_names)) parser.add_argument('--st_type', help='skipthoughts type') parser.add_argument('--st_dropout', type=float) parser.add_argument('--st_fixed_emb', default=None, type=utils.str2bool, help='backprop on embedding') # bert options parser.add_argument( '--bert_model', default="bert-base-multilingual-uncased", help= 'bert model: bert-base-uncased | bert-base-multilingual-uncased | bert-base-multilingual-cased' ) # image options parser.add_argument( '--is_augment_image', default='1', help='whether to augment images at the beginning of every epoch?') # optim options parser.add_argument('-lr', '--learning_rate', type=float, help='initial learning rate') parser.add_argument('-b', '--batch_size', type=int, help='mini-batch size') parser.add_argument('--epochs', type=int, help='number of total epochs to run') # options not in yaml file parser.add_argument('--start_epoch', default=0, type=int, help='manual epoch number (useful on restarts)') parser.add_argument('--resume', default='best', type=str, help='path to latest checkpoint') parser.add_argument('--save_model', default=True, type=utils.str2bool, help='able or disable save model and optim state') parser.add_argument( '--save_all_from', type=int, help='''delete the preceding checkpoint until an epoch,''' ''' then keep all (useful to save disk space)')''') parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', help='evaluate model on validation and test set', default=True) parser.add_argument('-j', '--workers', default=0, type=int, help='number of data loading workers') parser.add_argument('--print_freq', '-p', default=10, type=int, help='print frequency') ################################################ parser.add_argument('-ho', '--help_opt', dest='help_opt', action='store_true', help='show selected options before running') args = parser.parse_args() if vqa_trainsplit == "train": args.dir_logs = "logs/med/train/{}".format(model_name) else: args.dir_logs = "logs/med/trainval/{}".format(model_name) if "globalbilinear" in model_name: path_opt = model_name.replace("globalbilinear", "bilinear") if "_cased" in path_opt: path_opt = path_opt.replace("_cased", "") if "_uncased" in path_opt: path_opt = path_opt.replace("_uncased", "") elif "_cased_768" in model_name: path_opt = model_name.replace("_cased_768", "_768") elif "_uncased_768" in model_name: path_opt = model_name.replace("_uncased_768", "_768") elif "_cased" in model_name and "768" not in model_name: path_opt = model_name.replace("_cased", "") elif "_uncased" in model_name and "768" not in model_name: path_opt = model_name.replace("_uncased", "") else: path_opt = model_name path_opt = path_opt.replace("_trainval_", "_train_") args.path_opt = "{}/{}.yaml".format(args.dir_logs, path_opt) ######################################################################################### # Create options ######################################################################################### if "_cased" in model_name: args.bert_model = "bert-base-multilingual-cased" elif "_uncased" in model_name: args.bert_model = "bert-base-multilingual-uncased" if args.bert_model == "bert-base-uncased": question_features_path = BASE_EXTRACTED_QUES_FEATURES_PATH elif args.bert_model == "bert-base-multilingual-cased": question_features_path = CASED_EXTRACTED_QUES_FEATURES_PATH else: question_features_path = EXTRACTED_QUES_FEATURES_PATH options = { 'vqa': { 'trainsplit': args.vqa_trainsplit }, 'logs': { 'dir_logs': args.dir_logs }, 'model': { 'arch': args.arch, 'seq2vec': { 'type': args.st_type, 'dropout': args.st_dropout, 'fixed_emb': args.st_fixed_emb } }, 'optim': { 'lr': args.learning_rate, 'batch_size': args.batch_size, 'epochs': args.epochs } } if args.path_opt is not None: with open(args.path_opt, 'r') as handle: options_yaml = yaml.load(handle, Loader=yaml.FullLoader) options = utils.update_values(options, options_yaml) print('## args') pprint(vars(args)) print('## options') pprint(options) if args.help_opt: return # Set datasets options if 'vgenome' not in options: options['vgenome'] = None ######################################################################################### # Create needed datasets ######################################################################################### trainset = datasets.factory_VQA(options['vqa']['trainsplit'], options['vqa'], options['coco'], options['vgenome']) train_loader = trainset.data_loader( batch_size=options['optim']['batch_size'], num_workers=args.workers, shuffle=True) if options['vqa']['trainsplit'] == 'train': valset = datasets.factory_VQA('val', options['vqa'], options['coco']) val_loader = valset.data_loader( batch_size=options['optim']['batch_size'], num_workers=args.workers) if options['vqa']['trainsplit'] == 'trainval' or args.evaluate: testset = datasets.factory_VQA('test', options['vqa'], options['coco']) test_loader = testset.data_loader( batch_size=options['optim']['batch_size'], num_workers=args.workers) ######################################################################################### # Create model, criterion and optimizer ######################################################################################### model = models.factory(options['model'], trainset.vocab_words(), trainset.vocab_answers(), cuda=True, data_parallel=True) criterion = criterions.factory(options['vqa'], cuda=True) optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, model.parameters()), options['optim']['lr']) ######################################################################################### # args.resume: resume from a checkpoint OR create logs directory ######################################################################################### exp_logger = None if args.resume: args.start_epoch, best_acc1, exp_logger = load_checkpoint( model.module, optimizer, os.path.join(options['logs']['dir_logs'], args.resume)) else: # Or create logs directory if os.path.isdir(options['logs']['dir_logs']): if click.confirm( 'Logs directory already exists in {}. Erase?'.format( options['logs']['dir_logs'], default=False)): os.system('rm -r ' + options['logs']['dir_logs']) else: return os.system('mkdir -p ' + options['logs']['dir_logs']) path_new_opt = os.path.join(options['logs']['dir_logs'], os.path.basename(args.path_opt)) path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml') with open(path_new_opt, 'w') as f: yaml.dump(options, f, default_flow_style=False) with open(path_args, 'w') as f: yaml.dump(vars(args), f, default_flow_style=False) if exp_logger is None: # Set loggers exp_name = os.path.basename( options['logs']['dir_logs']) # add timestamp exp_logger = logger.Experiment(exp_name, options) exp_logger.add_meters('train', make_meters()) exp_logger.add_meters('test', make_meters()) if options['vqa']['trainsplit'] == 'train': exp_logger.add_meters('val', make_meters()) exp_logger.info['model_params'] = utils.params_count(model) print('Model has {} parameters'.format( exp_logger.info['model_params'])) ######################################################################################### # args.evaluate: on valset OR/AND on testset ######################################################################################### if args.evaluate: path_logger_json = os.path.join(options['logs']['dir_logs'], 'logger.json') if options['vqa']['trainsplit'] == 'train': acc1, val_results, prob = engine.validate( val_loader, model, criterion, exp_logger, args.start_epoch, args.print_freq, dict=io_utils.read_pickle(question_features_path), bert_dim=options["model"]["dim_q"], is_return_prob=True) else: test_results, testdev_results, prob = engine.test( test_loader, model, exp_logger, 1, args.print_freq, dict=io_utils.read_pickle(question_features_path), bert_dim=options["model"]["dim_q"], is_return_prob=True) torch.cuda.empty_cache() if vqa_trainsplit == "train": return prob, val_loader else: return prob, test_loader