def initialize(args, dataset="breast"):
options = {
'logs': {
'dir_logs': args.dir_logs
}
}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
print("\n>> load trainset...")
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'],
options['coco'],
options['vgenome'])
print("\n>> load cnn model...")
if dataset == "idrid":
cnn = convnets_idrid.factory(
{'arch': "resnet152_idrid"}, cuda=True, data_parallel=False)
elif dataset == "tools":
cnn = convnets_tools.factory(
{'arch': "resnet152_tools"}, cuda=True, data_parallel=False)
elif dataset == "breast":
cnn = convnets_breast.factory(
{'arch': "resnet152_breast"}, cuda=True, data_parallel=False)
cnn = cnn.cuda()
print("\n>> load vqa model...")
model = load_vqa_model(args, dataset, args.vqa_model)
model = model.cuda()
return cnn, model, trainset
def construct_dataloader(split: str,
options: dict) -> torch.utils.data.DataLoader:
dataset = datasets.factory_VQA(options["dataset"][split], split,
options["dataset"])
shuffle = split == "train"
dataset_loader = dataset.data_loader(
batch_size=options["optim"]["batch_size"],
num_workers=1,
shuffle=shuffle,
)
return dataset_loader
def get_model_vqa(vqa_model="minhmul_noatt_train_2048"):
path = "options/breast/{}.yaml".format(vqa_model)
args = parser.parse_args()
options = {
'vqa': {
'trainsplit': args.vqa_trainsplit
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
},
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs
}
}
with open(path, 'r') as handle:
options_yaml = yaml.load(handle)
options = vqa_utils.update_values(options, options_yaml)
if 'vgenome' not in options:
options['vgenome'] = None
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'],
options['coco'],
options['vgenome'])
model = models_vqa.factory(options['model'],
trainset.vocab_words(), trainset.vocab_answers(),
cuda=False, data_parallel=False)
# load checkpoint
path_ckpt_model = "logs/breast/{}/best_model.pth.tar".format(vqa_model)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if os.path.isfile(path_ckpt_model):
model_state = torch.load(path_ckpt_model, map_location=device)
model.load_state_dict(model_state)
return model
def main():
global args, options, model, cnn, transform, trainset
args = parser.parse_args()
options = {'logs': {'dir_logs': args.dir_logs}}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
print('## args')
pprint(vars(args))
print('## options')
pprint(options)
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'])
#options['coco'])
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
transform = transforms.Compose([
transforms.Scale(options['coco']['size']),
transforms.CenterCrop(options['coco']['size']),
transforms.ToTensor(),
normalize,
])
opt_factory_cnn = {'arch': options['coco']['arch']}
cnn = convnets.factory(opt_factory_cnn,
cuda=args.cuda,
data_parallel=False)
model = models.factory(options['model'],
trainset.vocab_words(),
trainset.vocab_answers(),
cuda=args.cuda,
data_parallel=False)
model.eval()
start_epoch, best_acc1, _ = load_checkpoint(
model, None, os.path.join(options['logs']['dir_logs'], args.resume))
my_local_ip = '192.168.0.32'
my_local_port = 3456
run_simple(my_local_ip, my_local_port, application)
def load_vqa_model(args, dataset, vqa_model="minhmul_noatt_train_2048"):
path = "options/{}/{}.yaml".format(dataset, vqa_model)
args = parser.parse_args()
options = {
'vqa': {
'trainsplit': args.vqa_trainsplit
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
}
}
with open(path, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
if 'vgenome' not in options:
options['vgenome'] = None
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'], options['coco'],
options['vgenome'])
model = models_vqa.factory(options['model'],
trainset.vocab_words(),
trainset.vocab_answers(),
cuda=False,
data_parallel=False)
# load checkpoint
path_ckpt_model = "logs/{}/{}/best_model.pth.tar".format(
dataset, vqa_model)
if os.path.isfile(path_ckpt_model):
model = load_dict_torch_031(model, path_ckpt_model)
return model
def main():
global args, best_acc1
args = parser.parse_args()
# Set options
options = {
'vqa' : {
'trainsplit': args.vqa_trainsplit,
'dropout': args.emb_drop
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
},
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs
}
}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
print('## args'); pprint(vars(args))
print('## options'); pprint(options)
if args.help_opt:
return
# Set datasets
trainset = datasets.factory_VQA(options['vqa']['trainsplit'], options['vqa'], options['coco'])
train_loader = trainset.data_loader(batch_size=options['optim']['batch_size'],
num_workers=args.workers,
shuffle=True)
if options['vqa']['trainsplit'] == 'train':
valset = datasets.factory_VQA('val', options['vqa'], options['coco'])
val_loader = valset.data_loader(batch_size=options['optim']['batch_size'],
num_workers=args.workers)
if options['vqa']['trainsplit'] == 'trainval' or args.evaluate:
testset = datasets.factory_VQA('test', options['vqa'], options['coco'])
test_loader = testset.data_loader(batch_size=options['optim']['batch_size'],
num_workers=args.workers)
# Set model, criterion and optimizer
model = getattr(models, options['model']['arch'])(
options['model'], trainset.vocab_words(), trainset.vocab_answers())
model = nn.DataParallel(model).cuda()
criterion = criterions.factory_loss(options['vqa'], cuda=True)
#optimizer = torch.optim.Adam([model.module.seq2vec.rnn.gru_cell.parameters()], options['optim']['lr'])
#optimizer = torch.optim.Adam(model.parameters(), options['optim']['lr'])
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), options['optim']['lr'])
# Optionally resume from a checkpoint
exp_logger = None
if args.resume:
args.start_epoch, best_acc1, exp_logger = load_checkpoint(model.module, optimizer,
os.path.join(options['logs']['dir_logs'], args.resume))
else:
# Or create logs directory
if os.path.isdir(options['logs']['dir_logs']):
if click.confirm('Logs directory already exists in {}. Erase?'
.format(options['logs']['dir_logs'], default=False)):
os.system('rm -r ' + options['logs']['dir_logs'])
else:
return
os.system('mkdir -p ' + options['logs']['dir_logs'])
path_new_opt = os.path.join(options['logs']['dir_logs'],
os.path.basename(args.path_opt))
path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml')
with open(path_new_opt, 'w') as f:
yaml.dump(options, f, default_flow_style=False)
with open(path_args, 'w') as f:
yaml.dump(vars(args), f, default_flow_style=False)
if exp_logger is None:
# Set loggers
exp_name = os.path.basename(options['logs']['dir_logs']) # add timestamp
exp_logger = logger.Experiment(exp_name, options)
exp_logger.add_meters('train', make_meters())
exp_logger.add_meters('test', make_meters())
if options['vqa']['trainsplit'] == 'train':
exp_logger.add_meters('val', make_meters())
exp_logger.info['model_params'] = utils.params_count(model)
print('Model has {} parameters'.format(exp_logger.info['model_params']))
# Begin evaluation and training
if args.evaluate:
path_logger_json = os.path.join(options['logs']['dir_logs'], 'logger.json')
if options['vqa']['trainsplit'] == 'train':
acc1, val_results = engine.validate(val_loader, model, criterion,
exp_logger, args.start_epoch, args.print_freq)
# save results and compute OpenEnd accuracy
exp_logger.to_json(path_logger_json)
save_results(val_results, args.start_epoch, valset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
test_results, testdev_results = engine.test(test_loader, model, exp_logger,
args.start_epoch, args.print_freq)
# save results and DOES NOT compute OpenEnd accuracy
exp_logger.to_json(path_logger_json)
save_results(test_results, args.start_epoch, testset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
save_results(testdev_results, args.start_epoch, testset.split_name(testdev=True),
options['logs']['dir_logs'], options['vqa']['dir'])
return
for epoch in range(args.start_epoch+1, options['optim']['epochs']):
#adjust_learning_rate(optimizer, epoch)
# train for one epoch
engine.train(train_loader, model, criterion, optimizer,
exp_logger, epoch, args.print_freq)
if options['vqa']['trainsplit'] == 'train':
# evaluate on validation set
acc1, val_results = engine.validate(val_loader, model, criterion,
exp_logger, epoch, args.print_freq)
# remember best prec@1 and save checkpoint
is_best = acc1 > best_acc1
best_acc1 = max(acc1, best_acc1)
save_checkpoint({
'epoch': epoch,
'arch': options['model']['arch'],
'best_acc1': best_acc1,
'exp_logger': exp_logger
},
model.module.state_dict(),
optimizer.state_dict(),
options['logs']['dir_logs'],
args.save_model,
args.save_all_from,
is_best)
# save results and compute OpenEnd accuracy
save_results(val_results, epoch, valset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
else:
test_results, testdev_results = engine.test(test_loader, model, exp_logger,
epoch, args.print_freq)
# save checkpoint at every timestep
save_checkpoint({
'epoch': epoch,
'arch': options['model']['arch'],
'best_acc1': best_acc1,
'exp_logger': exp_logger
},
model.module.state_dict(),
optimizer.state_dict(),
options['logs']['dir_logs'],
args.save_model,
args.save_all_from)
# save results and DOES NOT compute OpenEnd accuracy
save_results(test_results, epoch, testset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
save_results(testdev_results, epoch, testset.split_name(testdev=True),
options['logs']['dir_logs'], options['vqa']['dir'])
def main():
to_set_trainable = True
global args, best_acc1, best_acc5, best_acc10, best_loss_q
args = parser.parse_args()
# Set options
options = {
'vqa' : {
'trainsplit': args.vqa_trainsplit,
'partial': args.partial,
},
'logs': {
'dir_logs': args.dir_logs
},
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs,
'eval_epochs': args.eval_epochs,
}
}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
if 'model' not in options.keys():
options['model'] = {}
if args.dual_training:
options['logs']['dir_logs'] += '_dual_training'
print('## args'); pprint(vars(args))
print('## options'); pprint(options)
if args.help_opt:
return
# Set datasets
print('Loading dataset....',)
trainset = datasets.factory_VQA(options['vqa']['trainsplit'], options['vqa'],
opt_coco=options.get('coco', None),
opt_clevr=options.get('clevr', None),
opt_vgenome=options.get('vgnome', None))
train_loader = trainset.data_loader(batch_size=options['optim']['batch_size'],
num_workers=1,
shuffle=True)
if options['vqa'].get('sample_concept', False):
options['model']['concept_num'] = len(trainset.cid_to_concept)
valset = datasets.factory_VQA('val', options['vqa'],
opt_coco=options.get('coco', None),
opt_clevr=options.get('clevr', None),
opt_vgenome=options.get('vgnome', None))
val_loader = valset.data_loader(batch_size=options['optim']['batch_size'],
num_workers=1)
test_loader = valset.data_loader(batch_size=1 if args.beam_search else options['optim']['batch_size'],
num_workers=1)
print('Done.')
print('Setting up the model...')
# Set model, criterion and optimizer
# assert options['model']['arch_resnet'] == options['coco']['arch'], 'Two [arch] should be set the same.'
model = getattr(models, options['model']['arch'])(
options['model'], trainset.vocab_words(), trainset.vocab_answers())
if args.share_embeddings:
model.set_share_parameters()
# set_trainable(model.shared_conv_layer, False)
#optimizer = torch.optim.Adam([model.module.seq2vec.rnn.gru_cell.parameters()], options['optim']['lr'])
#optimizer = torch.optim.Adam(model.parameters(), options['optim']['lr'])
# optimizer = torch.optim.Adam([
# {'params': model.attention.parameters(), },
# {'params': model.seq2vec.parameters(), },
# {'params': model.vqa_module.parameters(), },
# {'params': list(model.vqg_module.parameters())[1:], 'weight_decay': 0.0},
# # filter(lambda p: p.requires_grad, model.parameters())
# ], lr=options['optim']['lr'], weight_decay=options['optim']['weight_decay'])
optimizer = torch.optim.Adam(params=filter(lambda p: p.requires_grad, model.parameters()),
lr=options['optim']['lr'], weight_decay=options['optim']['weight_decay'])
# Optionally resume from a checkpoint
exp_logger = None
if args.resume:
print('Loading saved model...')
args.start_epoch, best_acc1, exp_logger = load_checkpoint(model, optimizer,#model.module, optimizer,
os.path.join(options['logs']['dir_logs'], args.resume))
else:
# Or create logs directory
if os.path.isdir(options['logs']['dir_logs']):
if click.confirm('Logs directory already exists in {}. Erase?'
.format(options['logs']['dir_logs'], default=False)):
os.system('rm -r ' + options['logs']['dir_logs'])
else:
return
os.system('mkdir -p ' + options['logs']['dir_logs'])
path_new_opt = os.path.join(options['logs']['dir_logs'],
os.path.basename(args.path_opt))
path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml')
with open(path_new_opt, 'w') as f:
yaml.dump(options, f, default_flow_style=False)
with open(path_args, 'w') as f:
yaml.dump(vars(args), f, default_flow_style=False)
if exp_logger is None:
# Set loggers
exp_name = os.path.basename(options['logs']['dir_logs']) # add timestamp
exp_logger = logger.Experiment(exp_name, options)
exp_logger.add_meters('train', make_meters())
exp_logger.add_meters('test', make_meters())
if options['vqa']['trainsplit'] == 'train':
exp_logger.add_meters('val', make_meters())
exp_logger.info['model_params'] = utils.params_count(model)
print('Model has {} parameters'.format(exp_logger.info['model_params']))
# Begin evaluation and training
model = nn.DataParallel(model).cuda()
if args.evaluate:
print('Start evaluating...')
path_logger_json = os.path.join(options['logs']['dir_logs'], 'logger.json')
evaluate_result = engine.evaluate(test_loader, model, exp_logger, args.print_freq)
pdb.set_trace()
save_results(evaluate_result, args.start_epoch, valset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
return
print('Start training')
for epoch in range(args.start_epoch, options['optim']['epochs']):
#adjust_learning_rate(optimizer, epoch)selected_a.reinforce(reward.data.view(selected_a.size()))
# train for one epoch
# at first, the conv layers are fixed
# if epoch > args.finetuning_conv_epoch and to_set_trainable:
# set_trainable(model.module.shared_conv_layer, True)
# optimizer = select_optimizer(
# options['optim']['optimizer'], params=filter(lambda p: p.requires_grad, model.parameters()),
# lr=options['optim']['lr'], weight_decay=options['optim']['weight_decay'])
# to_set_trainable = False
# optimizer = adjust_optimizer(optimizer, epoch, regime)
engine.train(train_loader, model, optimizer,
exp_logger, epoch, args.print_freq,
dual_training=args.dual_training,
alternative_train=args.alternative_train)
if options['vqa']['trainsplit'] == 'train':
# evaluate on validation set
acc1, acc5, acc10, loss_q = engine.validate(test_loader, model,
exp_logger, epoch, args.print_freq)
if (epoch + 1) % options['optim']['eval_epochs'] == 0:
#print('[epoch {}] evaluation:'.format(epoch))
evaluate_result = engine.evaluate(test_loader, model, exp_logger, args.print_freq) #model.module, exp_logger, args.print_freq)
save_results(evaluate_result, epoch, valset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'], is_testing=False)
# remember best prec@1 and save checkpoint
is_best = acc1 > best_acc1
is_best_q = loss_q < best_loss_q
best_acc5 = acc5 if is_best else best_acc5
best_acc10 = acc10 if is_best else best_acc10
best_acc1 = acc1 if is_best else best_acc1
best_loss_q = loss_q if is_best_q else best_loss_q
print('** [Best]\tAcc@1: {0:.2f}%\tAcc@5: {1:.2f}%\tAcc@10: {2:.2f}% \tQ_Loss: {3:.4f}'.format(
best_acc1, best_acc5, best_acc10, best_loss_q))
save_checkpoint({
'epoch': epoch,
'arch': options['model']['arch'],
'best_acc1': best_acc1,
'best_acc5': best_acc5,
'best_acc10': best_acc10,
'exp_logger': exp_logger
},
model.module.state_dict(), #model.module.state_dict(),
optimizer.state_dict(),
options['logs']['dir_logs'],
args.save_model,
args.save_all_from,
is_best, is_best_q)
else:
raise NotImplementedError
def main():
global args, best_acc1
args = parser.parse_args()
#########################################################################################
# Create options
#########################################################################################
if args.bert_model == "bert-base-uncased":
question_features_path = BASE_EXTRACTED_QUES_FEATURES_PATH
elif args.bert_model == "bert-base-multilingual-cased":
question_features_path = CASED_EXTRACTED_QUES_FEATURES_PATH
else:
question_features_path = EXTRACTED_QUES_FEATURES_PATH
options = {
'vqa': {
'trainsplit': args.vqa_trainsplit
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
},
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs
}
}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle, Loader=yaml.FullLoader)
options = utils.update_values(options, options_yaml)
print('## args')
pprint(vars(args))
print('## options')
pprint(options)
if args.help_opt:
return
# Set datasets options
if 'vgenome' not in options:
options['vgenome'] = None
#########################################################################################
# Create needed datasets
#########################################################################################
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'], options['coco'],
options['vgenome'])
train_loader = trainset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers,
shuffle=True)
if options['vqa']['trainsplit'] == 'train':
valset = datasets.factory_VQA('val', options['vqa'], options['coco'])
val_loader = valset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers)
if options['vqa']['trainsplit'] == 'trainval' or args.evaluate:
testset = datasets.factory_VQA('test', options['vqa'], options['coco'])
test_loader = testset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers)
#########################################################################################
# Create model, criterion and optimizer
#########################################################################################
model = models.factory(options['model'],
trainset.vocab_words(),
trainset.vocab_answers(),
cuda=True,
data_parallel=True)
criterion = criterions.factory(options['vqa'], cuda=True)
optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()),
options['optim']['lr'])
#########################################################################################
# args.resume: resume from a checkpoint OR create logs directory
#########################################################################################
exp_logger = None
if args.resume:
args.start_epoch, best_acc1, exp_logger = load_checkpoint(
model.module, optimizer,
os.path.join(options['logs']['dir_logs'], args.resume))
else:
# Or create logs directory
if os.path.isdir(options['logs']['dir_logs']):
if click.confirm(
'Logs directory already exists in {}. Erase?'.format(
options['logs']['dir_logs'], default=False)):
os.system('rm -r ' + options['logs']['dir_logs'])
else:
return
os.system('mkdir -p ' + options['logs']['dir_logs'])
path_new_opt = os.path.join(options['logs']['dir_logs'],
os.path.basename(args.path_opt))
path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml')
with open(path_new_opt, 'w') as f:
yaml.dump(options, f, default_flow_style=False)
with open(path_args, 'w') as f:
yaml.dump(vars(args), f, default_flow_style=False)
if exp_logger is None:
# Set loggers
exp_name = os.path.basename(
options['logs']['dir_logs']) # add timestamp
exp_logger = logger.Experiment(exp_name, options)
exp_logger.add_meters('train', make_meters())
exp_logger.add_meters('test', make_meters())
if options['vqa']['trainsplit'] == 'train':
exp_logger.add_meters('val', make_meters())
exp_logger.info['model_params'] = utils.params_count(model)
print('Model has {} parameters'.format(
exp_logger.info['model_params']))
#########################################################################################
# args.evaluate: on valset OR/AND on testset
#########################################################################################
if args.evaluate:
path_logger_json = os.path.join(options['logs']['dir_logs'],
'logger.json')
if options['vqa']['trainsplit'] == 'train':
acc1, val_results = engine.validate(val_loader, model, criterion,
exp_logger, args.start_epoch,
args.print_freq)
# save results and compute OpenEnd accuracy
exp_logger.to_json(path_logger_json)
save_results(val_results, args.start_epoch, valset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
test_results, testdev_results = engine.test(test_loader, model,
exp_logger,
args.start_epoch,
args.print_freq)
# save results and DOES NOT compute OpenEnd accuracy
exp_logger.to_json(path_logger_json)
save_results(test_results, args.start_epoch, testset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
save_results(testdev_results, args.start_epoch,
testset.split_name(testdev=True),
options['logs']['dir_logs'], options['vqa']['dir'])
return
#########################################################################################
# Begin training on train/val or trainval/test
#########################################################################################
for epoch in range(args.start_epoch + 1, options['optim']['epochs']):
# if epoch > 1 and gen_utils.str2bool(args.is_augment_image) and 'options/med/' in args.path_opt:
# cmd = "python main/extract.py --dir_data data/raw/vqa_med/preprocessed --dataset med --is_augment_image 1 -b 64"
# os.system(cmd)
# if epoch == 1 and 'options/med/' in args.path_opt:
# cmd = "python main/extract.py --dir_data data/raw/vqa_med/preprocessed --dataset med --is_augment_image 0 -b 64"
# os.system(cmd)
# train for one epoch
engine.train(train_loader,
model,
criterion,
optimizer,
exp_logger,
epoch,
args.print_freq,
dict=io_utils.read_pickle(question_features_path),
bert_dim=options["model"]["dim_q"])
if options['vqa']['trainsplit'] == 'train':
# evaluate on validation set
acc1, val_results = engine.validate(
val_loader,
model,
criterion,
exp_logger,
epoch,
args.print_freq,
topk=5,
dict=io_utils.read_pickle(question_features_path),
bert_dim=options["model"]["dim_q"])
# remember best prec@1 and save checkpoint
is_best = acc1 > best_acc1
best_acc1 = max(acc1, best_acc1)
save_checkpoint(
{
'epoch': epoch,
'arch': options['model']['arch'],
'best_acc1': best_acc1,
'exp_logger': exp_logger
}, model.module.state_dict(), optimizer.state_dict(),
options['logs']['dir_logs'], args.save_model,
args.save_all_from, is_best)
# save results and compute OpenEnd accuracy
save_results(val_results, epoch, valset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
else:
test_results, testdev_results = engine.test(
test_loader,
model,
exp_logger,
epoch,
args.print_freq,
topk=5,
dict=io_utils.read_pickle(question_features_path),
bert_dim=options["model"]["dim_q"])
# save checkpoint at every timestep
save_checkpoint(
{
'epoch': epoch,
'arch': options['model']['arch'],
'best_acc1': best_acc1,
'exp_logger': exp_logger
}, model.module.state_dict(), optimizer.state_dict(),
options['logs']['dir_logs'], args.save_model,
args.save_all_from)
# save results and DOES NOT compute OpenEnd accuracy
save_results(test_results, epoch, testset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
save_results(testdev_results, epoch,
testset.split_name(testdev=True),
options['logs']['dir_logs'], options['vqa']['dir'])
def main():
global args, best_acc1
args = parser.parse_args()
#########################################################################################
# Create options
#########################################################################################
options = {
'vqa' : {
'trainsplit': args.vqa_trainsplit
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
},
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs
}
}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
# print('## args'); pprint(vars(args))
# print('## options'); pprint(options)
if args.help_opt:
return
# Set datasets options
if 'vgenome' not in options:
options['vgenome'] = None
#########################################################################################
# Create needed datasets
#########################################################################################
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'],
options['coco'],
options['vgenome'])
train_loader = trainset.data_loader(batch_size=options['optim']['batch_size'],
num_workers=args.workers,
shuffle=True)
imgs_trainset = CustomDataset(root_dir='FLIR/', transform=None)
imgs_train_loader = DataLoader(imgs_trainset, batch_size=4)
# if options['vqa']['trainsplit'] == 'train':
# valset = datasets.factory_VQA('val', options['vqa'], options['coco'])
# val_loader = valset.data_loader(batch_size=options['optim']['batch_size'],
# num_workers=args.workers)
if options['vqa']['trainsplit'] == 'trainval' or args.evaluate:
testset = datasets.factory_VQA('test', options['vqa'], options['coco'])
test_loader = testset.data_loader(batch_size=options['optim']['batch_size'],
num_workers=args.workers)
#########################################################################################
# Create model, criterion and optimizer
#########################################################################################
model = models.factory(options['model'],
trainset.vocab_words(), trainset.vocab_answers(),
cuda=True, data_parallel=True)
criterion = criterions.factory(options['vqa'], cuda=True)
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
options['optim']['lr'])
#########################################################################################
# args.resume: resume from a checkpoint OR create logs directory
#########################################################################################
exp_logger = None
if args.resume:
args.start_epoch, best_acc1, exp_logger = load_checkpoint(model.module, optimizer,
os.path.join(options['logs']['dir_logs'], args.resume))
# else:
# # Or create logs directory
# if os.path.isdir(options['logs']['dir_logs']):
# if click.confirm('Logs directory already exists in {}. Erase?'
# .format(options['logs']['dir_logs'], default=False)):
# os.system('rm -r ' + options['logs']['dir_logs'])
# else:
# return
# os.system('mkdir -p ' + options['logs']['dir_logs'])
# path_new_opt = os.path.join(options['logs']['dir_logs'],
# os.path.basename(args.path_opt))
# path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml')
# with open(path_new_opt, 'w') as f:
# yaml.dump(options, f, default_flow_style=False)
# with open(path_args, 'w') as f:
# yaml.dump(vars(args), f, default_flow_style=False)
if exp_logger is None:
# Set loggers
exp_name = os.path.basename(options['logs']['dir_logs']) # add timestamp
exp_logger = logger.Experiment(exp_name, options)
exp_logger.add_meters('train', make_meters())
exp_logger.add_meters('test', make_meters())
if options['vqa']['trainsplit'] == 'train':
exp_logger.add_meters('val', make_meters())
exp_logger.info['model_params'] = utils.params_count(model)
print('Model has {} parameters'.format(exp_logger.info['model_params']))
#########################################################################################
# args.evaluate: on valset OR/AND on testset
#########################################################################################
# if args.evaluate:
# path_logger_json = os.path.join(options['logs']['dir_logs'], 'logger.json')
# if options['vqa']['trainsplit'] == 'train':
# acc1, val_results = engine.validate(val_loader, model, criterion,
# exp_logger, args.start_epoch, args.print_freq)
# # save results and compute OpenEnd accuracy
# exp_logger.to_json(path_logger_json)
# save_results(val_results, args.start_epoch, valset.split_name(),
# options['logs']['dir_logs'], options['vqa']['dir'])
# test_results, testdev_results = engine.test(test_loader, model, exp_logger,
# args.start_epoch, args.print_freq)
# # save results and DOES NOT compute OpenEnd accuracy
# exp_logger.to_json(path_logger_json)
# save_results(test_results, args.start_epoch, testset.split_name(),
# options['logs']['dir_logs'], options['vqa']['dir'])
# save_results(testdev_results, args.start_epoch, testset.split_name(testdev=True),
# options['logs']['dir_logs'], options['vqa']['dir'])
# return
#########################################################################################
# Begin training on train/val or trainval/test
#########################################################################################
print("Started training")
for epoch in range(args.start_epoch+1, options['optim']['epochs']):
#adjust_learning_rate(optimizer, epoch)
# train for one epoch
engine.train(imgs_train_loader, model, criterion, optimizer,
exp_logger, epoch, args.print_freq)
def main():
global args, best_acc1
args = parser.parse_args()
#########################################################################################
# Create options
#########################################################################################
options = {
'vqa': {
'trainsplit': args.vqa_trainsplit
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
},
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs
}
}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
print('## args')
pprint(vars(args))
print('## options')
pprint(options)
if args.help_opt:
return
# Set datasets options
if 'vgenome' not in options:
options['vgenome'] = None
#########################################################################################
# Create needed datasets
#########################################################################################
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'], options['coco'],
options['vgenome'])
train_loader = trainset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers,
shuffle=True)
if options['vqa']['trainsplit'] == 'train':
valset = datasets.factory_VQA('val', options['vqa'], options['coco'])
val_loader = valset.data_loader(batch_size=2, num_workers=args.workers)
if options['vqa']['trainsplit'] == 'trainval' or args.evaluate:
testset = datasets.factory_VQA('test', options['vqa'], options['coco'])
test_loader = testset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers)
#########################################################################################
# Create model, criterion and optimizer
#########################################################################################
config = RagConfig.from_pretrained("facebook/rag-token-nq")
config.index_name = "legacy"
config.use_dummy_dataset = False
config.question_encoder.return_dict = True
config.n_docs = 10
# config.n_docs = 15
# import pdb;
# pdb.set_trace ()
if not args.evaluate and not args.resume:
tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-base",
config=config)
retriever = RagRetriever.from_pretrained("facebook/rag-token-base",
config=config)
model = RagTokenForGeneration.from_pretrained(
"facebook/rag-token-base", retriever=retriever, config=config)
else:
tokenizer = RagTokenizer.from_pretrained(os.path.join(
options['logs']['dir_logs'], "epoch_{}".format(args.start_epoch)),
config=config)
retriever = RagRetriever.from_pretrained(os.path.join(
options['logs']['dir_logs'], "epoch_{}".format(args.start_epoch)),
config=config)
model = RagTokenForGeneration.from_pretrained(os.path.join(
options['logs']['dir_logs'], "epoch_{}".format(args.start_epoch)),
retriever=retriever,
config=config)
model.cuda()
criterion = criterions.factory(options['vqa'], cuda=True)
no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
"weight_decay":
0.0,
},
{
"params": [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
"weight_decay":
0.0,
},
]
optimizer = AdamW(optimizer_grouped_parameters,
lr=options['optim']['lr'],
eps=1e-8)
# optimizer = torch.optim.SGD(optimizer_grouped_parameters, lr=options['optim']['lr'], momentum=0.9)
#########################################################################################
# args.resume: resume from a checkpoint OR create logs directory
#########################################################################################
exp_logger = None
# Or create logs directory
# os.system('mkdir -p ' + options['logs']['dir_logs'])
path_new_opt = os.path.join(options['logs']['dir_logs'],
os.path.basename(args.path_opt))
path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml')
with open(path_new_opt, 'w') as f:
yaml.dump(options, f, default_flow_style=False)
with open(path_args, 'w') as f:
yaml.dump(vars(args), f, default_flow_style=False)
if exp_logger is None:
# Set loggers
exp_name = os.path.basename(
options['logs']['dir_logs']) # add timestamp
exp_logger = logger.Experiment(exp_name, options)
exp_logger.add_meters('train', make_meters())
exp_logger.add_meters('test', make_meters())
if options['vqa']['trainsplit'] == 'train':
exp_logger.add_meters('val', make_meters())
exp_logger.info['model_params'] = utils.params_count(model)
print('Model has {} parameters'.format(
exp_logger.info['model_params']))
#########################################################################################
# args.evaluate: on valset OR/AND on testset
#########################################################################################
if args.evaluate:
path_logger_json = os.path.join(options['logs']['dir_logs'],
'logger.json')
if options['vqa']['trainsplit'] == 'train':
acc1, val_results = engine.validate(val_loader, model, retriever,
tokenizer, criterion,
exp_logger, args.start_epoch,
100)
# save results and compute OpenEnd accuracy
exp_logger.to_json(path_logger_json)
save_results(val_results, args.start_epoch, valset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
return
else:
for epoch in range(args.start_epoch + 1, options['optim']['epochs']):
engine.train(train_loader, model, retriever, tokenizer, criterion,
optimizer, exp_logger, epoch, args.print_freq)
# remember best prec@1 and save checkpoint
is_best = True
best_accs1 = -1
save_checkpoint(
{
'epoch': epoch,
'arch': options['model']['arch'],
'best_acc1': best_acc1,
'exp_logger': exp_logger
}, model, tokenizer, retriever, options['logs']['dir_logs'],
args.save_model, True)
def compute_prob_one_model(model_name, vqa_trainsplit="train"):
parser = argparse.ArgumentParser(
description='Train/Evaluate models',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
##################################################
# yaml options file contains all default choices #
# parser.add_argument('--path_opt', default='options/breast/default.yaml', type=str,
# help='path to a yaml options file')
parser.add_argument(
'--path_opt',
default='options/med/bilinear_att_train_imagenet_h200_g4.yaml',
type=str,
help='path to a yaml options file')
################################################
# change cli options to modify default choices #
# logs options
parser.add_argument(
'--dir_logs',
default='logs/med/train/globalbilinear_att_train_imagenet_h200_g4',
type=str,
help='dir logs')
# data options
parser.add_argument('--vqa_trainsplit',
type=str,
choices=['train', 'trainval'],
default=vqa_trainsplit)
# model options
parser.add_argument('--arch',
choices=models.model_names,
help='vqa model architecture: ' +
' | '.join(models.model_names))
parser.add_argument('--st_type', help='skipthoughts type')
parser.add_argument('--st_dropout', type=float)
parser.add_argument('--st_fixed_emb',
default=None,
type=utils.str2bool,
help='backprop on embedding')
# bert options
parser.add_argument(
'--bert_model',
default="bert-base-multilingual-uncased",
help=
'bert model: bert-base-uncased | bert-base-multilingual-uncased | bert-base-multilingual-cased'
)
# image options
parser.add_argument(
'--is_augment_image',
default='1',
help='whether to augment images at the beginning of every epoch?')
# optim options
parser.add_argument('-lr',
'--learning_rate',
type=float,
help='initial learning rate')
parser.add_argument('-b', '--batch_size', type=int, help='mini-batch size')
parser.add_argument('--epochs',
type=int,
help='number of total epochs to run')
# options not in yaml file
parser.add_argument('--start_epoch',
default=0,
type=int,
help='manual epoch number (useful on restarts)')
parser.add_argument('--resume',
default='best',
type=str,
help='path to latest checkpoint')
parser.add_argument('--save_model',
default=True,
type=utils.str2bool,
help='able or disable save model and optim state')
parser.add_argument(
'--save_all_from',
type=int,
help='''delete the preceding checkpoint until an epoch,'''
''' then keep all (useful to save disk space)')''')
parser.add_argument('-e',
'--evaluate',
dest='evaluate',
action='store_true',
help='evaluate model on validation and test set',
default=True)
parser.add_argument('-j',
'--workers',
default=0,
type=int,
help='number of data loading workers')
parser.add_argument('--print_freq',
'-p',
default=10,
type=int,
help='print frequency')
################################################
parser.add_argument('-ho',
'--help_opt',
dest='help_opt',
action='store_true',
help='show selected options before running')
args = parser.parse_args()
if vqa_trainsplit == "train":
args.dir_logs = "logs/med/train/{}".format(model_name)
else:
args.dir_logs = "logs/med/trainval/{}".format(model_name)
if "globalbilinear" in model_name:
path_opt = model_name.replace("globalbilinear", "bilinear")
if "_cased" in path_opt:
path_opt = path_opt.replace("_cased", "")
if "_uncased" in path_opt:
path_opt = path_opt.replace("_uncased", "")
elif "_cased_768" in model_name:
path_opt = model_name.replace("_cased_768", "_768")
elif "_uncased_768" in model_name:
path_opt = model_name.replace("_uncased_768", "_768")
elif "_cased" in model_name and "768" not in model_name:
path_opt = model_name.replace("_cased", "")
elif "_uncased" in model_name and "768" not in model_name:
path_opt = model_name.replace("_uncased", "")
else:
path_opt = model_name
path_opt = path_opt.replace("_trainval_", "_train_")
args.path_opt = "{}/{}.yaml".format(args.dir_logs, path_opt)
#########################################################################################
# Create options
#########################################################################################
if "_cased" in model_name:
args.bert_model = "bert-base-multilingual-cased"
elif "_uncased" in model_name:
args.bert_model = "bert-base-multilingual-uncased"
if args.bert_model == "bert-base-uncased":
question_features_path = BASE_EXTRACTED_QUES_FEATURES_PATH
elif args.bert_model == "bert-base-multilingual-cased":
question_features_path = CASED_EXTRACTED_QUES_FEATURES_PATH
else:
question_features_path = EXTRACTED_QUES_FEATURES_PATH
options = {
'vqa': {
'trainsplit': args.vqa_trainsplit
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
},
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs
}
}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle, Loader=yaml.FullLoader)
options = utils.update_values(options, options_yaml)
print('## args')
pprint(vars(args))
print('## options')
pprint(options)
if args.help_opt:
return
# Set datasets options
if 'vgenome' not in options:
options['vgenome'] = None
#########################################################################################
# Create needed datasets
#########################################################################################
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'], options['coco'],
options['vgenome'])
train_loader = trainset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers,
shuffle=True)
if options['vqa']['trainsplit'] == 'train':
valset = datasets.factory_VQA('val', options['vqa'], options['coco'])
val_loader = valset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers)
if options['vqa']['trainsplit'] == 'trainval' or args.evaluate:
testset = datasets.factory_VQA('test', options['vqa'], options['coco'])
test_loader = testset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers)
#########################################################################################
# Create model, criterion and optimizer
#########################################################################################
model = models.factory(options['model'],
trainset.vocab_words(),
trainset.vocab_answers(),
cuda=True,
data_parallel=True)
criterion = criterions.factory(options['vqa'], cuda=True)
optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()),
options['optim']['lr'])
#########################################################################################
# args.resume: resume from a checkpoint OR create logs directory
#########################################################################################
exp_logger = None
if args.resume:
args.start_epoch, best_acc1, exp_logger = load_checkpoint(
model.module, optimizer,
os.path.join(options['logs']['dir_logs'], args.resume))
else:
# Or create logs directory
if os.path.isdir(options['logs']['dir_logs']):
if click.confirm(
'Logs directory already exists in {}. Erase?'.format(
options['logs']['dir_logs'], default=False)):
os.system('rm -r ' + options['logs']['dir_logs'])
else:
return
os.system('mkdir -p ' + options['logs']['dir_logs'])
path_new_opt = os.path.join(options['logs']['dir_logs'],
os.path.basename(args.path_opt))
path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml')
with open(path_new_opt, 'w') as f:
yaml.dump(options, f, default_flow_style=False)
with open(path_args, 'w') as f:
yaml.dump(vars(args), f, default_flow_style=False)
if exp_logger is None:
# Set loggers
exp_name = os.path.basename(
options['logs']['dir_logs']) # add timestamp
exp_logger = logger.Experiment(exp_name, options)
exp_logger.add_meters('train', make_meters())
exp_logger.add_meters('test', make_meters())
if options['vqa']['trainsplit'] == 'train':
exp_logger.add_meters('val', make_meters())
exp_logger.info['model_params'] = utils.params_count(model)
print('Model has {} parameters'.format(
exp_logger.info['model_params']))
#########################################################################################
# args.evaluate: on valset OR/AND on testset
#########################################################################################
if args.evaluate:
path_logger_json = os.path.join(options['logs']['dir_logs'],
'logger.json')
if options['vqa']['trainsplit'] == 'train':
acc1, val_results, prob = engine.validate(
val_loader,
model,
criterion,
exp_logger,
args.start_epoch,
args.print_freq,
dict=io_utils.read_pickle(question_features_path),
bert_dim=options["model"]["dim_q"],
is_return_prob=True)
else:
test_results, testdev_results, prob = engine.test(
test_loader,
model,
exp_logger,
1,
args.print_freq,
dict=io_utils.read_pickle(question_features_path),
bert_dim=options["model"]["dim_q"],
is_return_prob=True)
torch.cuda.empty_cache()
if vqa_trainsplit == "train":
return prob, val_loader
else:
return prob, test_loader
