def get_model_vqa(vqa_model="minhmul_noatt_train_2048"):
path = "options/breast/{}.yaml".format(vqa_model)
args = parser.parse_args()
options = {
'vqa': {
'trainsplit': args.vqa_trainsplit
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
},
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs
}
}
with open(path, 'r') as handle:
options_yaml = yaml.load(handle)
options = vqa_utils.update_values(options, options_yaml)
if 'vgenome' not in options:
options['vgenome'] = None
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'],
options['coco'],
options['vgenome'])
model = models_vqa.factory(options['model'],
trainset.vocab_words(), trainset.vocab_answers(),
cuda=False, data_parallel=False)
# load checkpoint
path_ckpt_model = "logs/breast/{}/best_model.pth.tar".format(vqa_model)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if os.path.isfile(path_ckpt_model):
model_state = torch.load(path_ckpt_model, map_location=device)
model.load_state_dict(model_state)
return model
def main():
global args, options, model, cnn, transform, trainset
args = parser.parse_args()
options = {'logs': {'dir_logs': args.dir_logs}}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
print('## args')
pprint(vars(args))
print('## options')
pprint(options)
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'])
#options['coco'])
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
transform = transforms.Compose([
transforms.Scale(options['coco']['size']),
transforms.CenterCrop(options['coco']['size']),
transforms.ToTensor(),
normalize,
])
opt_factory_cnn = {'arch': options['coco']['arch']}
cnn = convnets.factory(opt_factory_cnn,
cuda=args.cuda,
data_parallel=False)
model = models.factory(options['model'],
trainset.vocab_words(),
trainset.vocab_answers(),
cuda=args.cuda,
data_parallel=False)
model.eval()
start_epoch, best_acc1, _ = load_checkpoint(
model, None, os.path.join(options['logs']['dir_logs'], args.resume))
my_local_ip = '192.168.0.32'
my_local_port = 3456
run_simple(my_local_ip, my_local_port, application)
def load_vqa_model(args, dataset, vqa_model="minhmul_noatt_train_2048"):
path = "options/{}/{}.yaml".format(dataset, vqa_model)
args = parser.parse_args()
options = {
'vqa': {
'trainsplit': args.vqa_trainsplit
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
}
}
with open(path, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
if 'vgenome' not in options:
options['vgenome'] = None
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'], options['coco'],
options['vgenome'])
model = models_vqa.factory(options['model'],
trainset.vocab_words(),
trainset.vocab_answers(),
cuda=False,
data_parallel=False)
# load checkpoint
path_ckpt_model = "logs/{}/{}/best_model.pth.tar".format(
dataset, vqa_model)
if os.path.isfile(path_ckpt_model):
model = load_dict_torch_031(model, path_ckpt_model)
return model
def main():
global args, best_acc1
args = parser.parse_args()
#########################################################################################
# Create options
#########################################################################################
if args.bert_model == "bert-base-uncased":
question_features_path = BASE_EXTRACTED_QUES_FEATURES_PATH
elif args.bert_model == "bert-base-multilingual-cased":
question_features_path = CASED_EXTRACTED_QUES_FEATURES_PATH
else:
question_features_path = EXTRACTED_QUES_FEATURES_PATH
options = {
'vqa': {
'trainsplit': args.vqa_trainsplit
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
},
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs
}
}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle, Loader=yaml.FullLoader)
options = utils.update_values(options, options_yaml)
print('## args')
pprint(vars(args))
print('## options')
pprint(options)
if args.help_opt:
return
# Set datasets options
if 'vgenome' not in options:
options['vgenome'] = None
#########################################################################################
# Create needed datasets
#########################################################################################
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'], options['coco'],
options['vgenome'])
train_loader = trainset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers,
shuffle=True)
if options['vqa']['trainsplit'] == 'train':
valset = datasets.factory_VQA('val', options['vqa'], options['coco'])
val_loader = valset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers)
if options['vqa']['trainsplit'] == 'trainval' or args.evaluate:
testset = datasets.factory_VQA('test', options['vqa'], options['coco'])
test_loader = testset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers)
#########################################################################################
# Create model, criterion and optimizer
#########################################################################################
model = models.factory(options['model'],
trainset.vocab_words(),
trainset.vocab_answers(),
cuda=True,
data_parallel=True)
criterion = criterions.factory(options['vqa'], cuda=True)
optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()),
options['optim']['lr'])
#########################################################################################
# args.resume: resume from a checkpoint OR create logs directory
#########################################################################################
exp_logger = None
if args.resume:
args.start_epoch, best_acc1, exp_logger = load_checkpoint(
model.module, optimizer,
os.path.join(options['logs']['dir_logs'], args.resume))
else:
# Or create logs directory
if os.path.isdir(options['logs']['dir_logs']):
if click.confirm(
'Logs directory already exists in {}. Erase?'.format(
options['logs']['dir_logs'], default=False)):
os.system('rm -r ' + options['logs']['dir_logs'])
else:
return
os.system('mkdir -p ' + options['logs']['dir_logs'])
path_new_opt = os.path.join(options['logs']['dir_logs'],
os.path.basename(args.path_opt))
path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml')
with open(path_new_opt, 'w') as f:
yaml.dump(options, f, default_flow_style=False)
with open(path_args, 'w') as f:
yaml.dump(vars(args), f, default_flow_style=False)
if exp_logger is None:
# Set loggers
exp_name = os.path.basename(
options['logs']['dir_logs']) # add timestamp
exp_logger = logger.Experiment(exp_name, options)
exp_logger.add_meters('train', make_meters())
exp_logger.add_meters('test', make_meters())
if options['vqa']['trainsplit'] == 'train':
exp_logger.add_meters('val', make_meters())
exp_logger.info['model_params'] = utils.params_count(model)
print('Model has {} parameters'.format(
exp_logger.info['model_params']))
#########################################################################################
# args.evaluate: on valset OR/AND on testset
#########################################################################################
if args.evaluate:
path_logger_json = os.path.join(options['logs']['dir_logs'],
'logger.json')
if options['vqa']['trainsplit'] == 'train':
acc1, val_results = engine.validate(val_loader, model, criterion,
exp_logger, args.start_epoch,
args.print_freq)
# save results and compute OpenEnd accuracy
exp_logger.to_json(path_logger_json)
save_results(val_results, args.start_epoch, valset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
test_results, testdev_results = engine.test(test_loader, model,
exp_logger,
args.start_epoch,
args.print_freq)
# save results and DOES NOT compute OpenEnd accuracy
exp_logger.to_json(path_logger_json)
save_results(test_results, args.start_epoch, testset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
save_results(testdev_results, args.start_epoch,
testset.split_name(testdev=True),
options['logs']['dir_logs'], options['vqa']['dir'])
return
#########################################################################################
# Begin training on train/val or trainval/test
#########################################################################################
for epoch in range(args.start_epoch + 1, options['optim']['epochs']):
# if epoch > 1 and gen_utils.str2bool(args.is_augment_image) and 'options/med/' in args.path_opt:
# cmd = "python main/extract.py --dir_data data/raw/vqa_med/preprocessed --dataset med --is_augment_image 1 -b 64"
# os.system(cmd)
# if epoch == 1 and 'options/med/' in args.path_opt:
# cmd = "python main/extract.py --dir_data data/raw/vqa_med/preprocessed --dataset med --is_augment_image 0 -b 64"
# os.system(cmd)
# train for one epoch
engine.train(train_loader,
model,
criterion,
optimizer,
exp_logger,
epoch,
args.print_freq,
dict=io_utils.read_pickle(question_features_path),
bert_dim=options["model"]["dim_q"])
if options['vqa']['trainsplit'] == 'train':
# evaluate on validation set
acc1, val_results = engine.validate(
val_loader,
model,
criterion,
exp_logger,
epoch,
args.print_freq,
topk=5,
dict=io_utils.read_pickle(question_features_path),
bert_dim=options["model"]["dim_q"])
# remember best prec@1 and save checkpoint
is_best = acc1 > best_acc1
best_acc1 = max(acc1, best_acc1)
save_checkpoint(
{
'epoch': epoch,
'arch': options['model']['arch'],
'best_acc1': best_acc1,
'exp_logger': exp_logger
}, model.module.state_dict(), optimizer.state_dict(),
options['logs']['dir_logs'], args.save_model,
args.save_all_from, is_best)
# save results and compute OpenEnd accuracy
save_results(val_results, epoch, valset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
else:
test_results, testdev_results = engine.test(
test_loader,
model,
exp_logger,
epoch,
args.print_freq,
topk=5,
dict=io_utils.read_pickle(question_features_path),
bert_dim=options["model"]["dim_q"])
# save checkpoint at every timestep
save_checkpoint(
{
'epoch': epoch,
'arch': options['model']['arch'],
'best_acc1': best_acc1,
'exp_logger': exp_logger
}, model.module.state_dict(), optimizer.state_dict(),
options['logs']['dir_logs'], args.save_model,
args.save_all_from)
# save results and DOES NOT compute OpenEnd accuracy
save_results(test_results, epoch, testset.split_name(),
options['logs']['dir_logs'], options['vqa']['dir'])
save_results(testdev_results, epoch,
testset.split_name(testdev=True),
options['logs']['dir_logs'], options['vqa']['dir'])
def main():
global args, best_acc1
args = parser.parse_args()
#########################################################################################
# Create options
#########################################################################################
options = {
'vqa' : {
'trainsplit': args.vqa_trainsplit
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
},
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs
}
}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
# print('## args'); pprint(vars(args))
# print('## options'); pprint(options)
if args.help_opt:
return
# Set datasets options
if 'vgenome' not in options:
options['vgenome'] = None
#########################################################################################
# Create needed datasets
#########################################################################################
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'],
options['coco'],
options['vgenome'])
train_loader = trainset.data_loader(batch_size=options['optim']['batch_size'],
num_workers=args.workers,
shuffle=True)
imgs_trainset = CustomDataset(root_dir='FLIR/', transform=None)
imgs_train_loader = DataLoader(imgs_trainset, batch_size=4)
# if options['vqa']['trainsplit'] == 'train':
# valset = datasets.factory_VQA('val', options['vqa'], options['coco'])
# val_loader = valset.data_loader(batch_size=options['optim']['batch_size'],
# num_workers=args.workers)
if options['vqa']['trainsplit'] == 'trainval' or args.evaluate:
testset = datasets.factory_VQA('test', options['vqa'], options['coco'])
test_loader = testset.data_loader(batch_size=options['optim']['batch_size'],
num_workers=args.workers)
#########################################################################################
# Create model, criterion and optimizer
#########################################################################################
model = models.factory(options['model'],
trainset.vocab_words(), trainset.vocab_answers(),
cuda=True, data_parallel=True)
criterion = criterions.factory(options['vqa'], cuda=True)
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
options['optim']['lr'])
#########################################################################################
# args.resume: resume from a checkpoint OR create logs directory
#########################################################################################
exp_logger = None
if args.resume:
args.start_epoch, best_acc1, exp_logger = load_checkpoint(model.module, optimizer,
os.path.join(options['logs']['dir_logs'], args.resume))
# else:
# # Or create logs directory
# if os.path.isdir(options['logs']['dir_logs']):
# if click.confirm('Logs directory already exists in {}. Erase?'
# .format(options['logs']['dir_logs'], default=False)):
# os.system('rm -r ' + options['logs']['dir_logs'])
# else:
# return
# os.system('mkdir -p ' + options['logs']['dir_logs'])
# path_new_opt = os.path.join(options['logs']['dir_logs'],
# os.path.basename(args.path_opt))
# path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml')
# with open(path_new_opt, 'w') as f:
# yaml.dump(options, f, default_flow_style=False)
# with open(path_args, 'w') as f:
# yaml.dump(vars(args), f, default_flow_style=False)
if exp_logger is None:
# Set loggers
exp_name = os.path.basename(options['logs']['dir_logs']) # add timestamp
exp_logger = logger.Experiment(exp_name, options)
exp_logger.add_meters('train', make_meters())
exp_logger.add_meters('test', make_meters())
if options['vqa']['trainsplit'] == 'train':
exp_logger.add_meters('val', make_meters())
exp_logger.info['model_params'] = utils.params_count(model)
print('Model has {} parameters'.format(exp_logger.info['model_params']))
#########################################################################################
# args.evaluate: on valset OR/AND on testset
#########################################################################################
# if args.evaluate:
# path_logger_json = os.path.join(options['logs']['dir_logs'], 'logger.json')
# if options['vqa']['trainsplit'] == 'train':
# acc1, val_results = engine.validate(val_loader, model, criterion,
# exp_logger, args.start_epoch, args.print_freq)
# # save results and compute OpenEnd accuracy
# exp_logger.to_json(path_logger_json)
# save_results(val_results, args.start_epoch, valset.split_name(),
# options['logs']['dir_logs'], options['vqa']['dir'])
# test_results, testdev_results = engine.test(test_loader, model, exp_logger,
# args.start_epoch, args.print_freq)
# # save results and DOES NOT compute OpenEnd accuracy
# exp_logger.to_json(path_logger_json)
# save_results(test_results, args.start_epoch, testset.split_name(),
# options['logs']['dir_logs'], options['vqa']['dir'])
# save_results(testdev_results, args.start_epoch, testset.split_name(testdev=True),
# options['logs']['dir_logs'], options['vqa']['dir'])
# return
#########################################################################################
# Begin training on train/val or trainval/test
#########################################################################################
print("Started training")
for epoch in range(args.start_epoch+1, options['optim']['epochs']):
#adjust_learning_rate(optimizer, epoch)
# train for one epoch
engine.train(imgs_train_loader, model, criterion, optimizer,
exp_logger, epoch, args.print_freq)
def main():
args = parser.parse_args()
#########################################################################################
# Create options
#########################################################################################
options = {
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs
}
}
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
options['vgenome'] = None
#########################################################################################
# Bookkeeping
#########################################################################################
if args.resume:
run_name = args.resume
save_dir = os.path.join('logs', 'cx', run_name)
assert(os.path.isdir(save_dir))
i = 1
log_dir = os.path.join('runs', run_name, 'resume_{}'.format(i))
while(os.path.isdir(log_dir)):
i += 1
log_dir = os.path.join('runs', run_name, 'resume_{}'.format(i))
else:
run_name = datetime.now().strftime('%b%d-%H-%M-%S')
if args.comment:
run_name += '_' + args.comment
save_dir = os.path.join('logs', 'cx', run_name)
if os.path.isdir(save_dir):
if click.confirm('Save directory already exists in {}. Erase?'.format(save_dir)):
os.system('rm -r ' + save_dir)
else:
return
os.makedirs(os.path.join(save_dir, 'ckpt'))
os.makedirs(os.path.join(save_dir, 'best'))
# Tensorboard log directory
log_dir = os.path.join('runs', run_name)
train_writer = SummaryWriter(log_dir=os.path.join(log_dir, 'train'))
val_writer = SummaryWriter(log_dir=os.path.join(log_dir, 'val'))
print('Saving model to {}'.format(save_dir))
print('Logging results to {}'.format(log_dir))
#########################################################################################
# Create datasets
#########################################################################################
print('=> Loading VQA dataset...')
if args.dev_mode:
trainset_fname = 'trainset_augmented_small.pickle'
else:
trainset_fname = 'trainset_augmented.pickle'
trainset = pickle.load(open(os.path.join(options['vqa']['path_trainset'], 'pickle_old', trainset_fname), 'rb'))
# if not args.dev_mode:
valset_fname = 'valset_augmented_small.pickle'
valset = pickle.load(open(os.path.join(options['vqa']['path_trainset'], 'pickle_old', valset_fname), 'rb'))
print('=> Loading KNN data...')
knns = json.load(open(options['coco']['path_knn'], 'r'))
knns = {int(k):v for k,v in knns.items()}
print('=> Loading COCO image features...')
features_train = h5py.File(os.path.join(options['coco']['path_raw'], 'trainset.hdf5'), 'r').get('noatt')
features_train = np.array(features_train)
# if not args.dev_mode:
features_val = h5py.File(os.path.join(options['coco']['path_raw'], 'valset.hdf5'), 'r').get('noatt')
features_val = np.array(features_val)
#########################################################################################
# Create model
#########################################################################################
print('=> Building model...')
vqa_model = models.factory(options['model'],
trainset['vocab_words'], trainset['vocab_answers'],
cuda=True, data_parallel=True)
vqa_model = vqa_model.module
if args.pretrained_vqa:
load_vqa_checkpoint(vqa_model, None, os.path.join(options['logs']['dir_logs'], 'best'))
cx_model = ContrastiveModel(vqa_model, knn_size=2, trainable_vqa=args.trainable_vqa)
optimizer = torch.optim.Adam(cx_model.parameters(), lr=options['optim']['lr'])
if args.resume:
info, start_epoch, best_recall = load_cx_checkpoint(cx_model, save_dir, resume_best=args.best)
else:
info = []
start_epoch = 1
best_recall = 0
cx_model.cuda()
#########################################################################################
# Train loop
#########################################################################################
print('=> Starting training...')
if args.pairwise:
print('==> Pairwise training')
for epoch in range(start_epoch, options['optim']['epochs'] + 1):
cx_model.train()
if vqa_model is not None:
if args.trainable_vqa:
vqa_model.train()
else:
vqa_model.eval()
train_b = 0
criterion = ContrastiveLoss()
criterion.cuda()
trainset_batched = batchify(trainset['examples_list'], batch_size=options['optim']['batch_size'])
for batch in tqdm(trainset_batched):
assert(cx_model.training)
if vqa_model is not None:
if args.trainable_vqa:
assert(vqa_model.training)
else:
assert(not vqa_model.training)
image_features, question_wids, answer_aids, comp_idxs = getDataFromBatch(batch, features_train, trainset['name_to_index'], pairwise=args.pairwise)
h_out = cx_model(image_features, question_wids, answer_aids)
loss_comp = criterion(h_out[:, 0], h_out[:, 1], label=Variable(torch.ones([len(batch)])).cuda())
loss_other = criterion(h_out[:, 0], h_out[:, 2], label=Variable(torch.zeros([len(batch)])).cuda())
loss = loss_comp + loss_other
if optimizer is not None:
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_b += 1
if train_b % args.print_freq == 0:
dists = cx_model.get_scores(h_out[:, 0], h_out[:, 1:])
dist_comp = dists[:, 0].mean()
dist_other = dists[:, 1].mean()
metrics = {
'contrastive/loss_comp': float(loss_comp),
'contrastive/loss_other': float(loss_other),
'contrastive/loss': float(loss),
'contrastive/dist_comp': float(dist_comp),
'contrastive/dist_other': float(dist_other)
}
log_results(train_writer, mode='train', epoch=epoch, i=((epoch - 1) * len(trainset_batched)) + train_b, metrics=metrics)
if (args.eval_freq > 0 and train_b % args.eval_freq == 0) or train_b == len(trainset_batched):
eval_results = eval_model(cx_model, valset, features_val, options['optim']['batch_size'], pairwise=args.pairwise)
log_results(val_writer, mode='val', epoch=epoch, i=((epoch - 1) * len(trainset_batched)) + train_b, metrics=eval_results)
info.append(eval_results)
if eval_results['contrastive/recall'] > best_recall:
is_best = True
best_recall = eval_results['contrastive/recall']
else:
is_best = False
save_cx_checkpoint(cx_model, info, save_dir, is_best=is_best)
eval_results = eval_model(cx_model, valset, features_val, options['optim']['batch_size'], pairwise=args.pairwise)
log_results(val_writer, mode='test', epoch=0, i=0, metrics=eval_results)
def main():
args = parser.parse_args()
##########################################################################
# Create options
##########################################################################
# Parse options
options = {
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs,
},
'cx_model': {
'pretrained_vqa': args.pretrained_vqa,
'trainable_vqa': args.trainable_vqa,
}
}
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle)
options = utils.update_values(options, options_yaml)
options['vgenome'] = None
# Ensure determinism
random.seed(42)
torch.manual_seed(42)
torch.cuda.manual_seed_all(42)
# Bookkeeping
##########################################################################
if args.cx_model == 'NeuralModel' and not args.comment:
args.comment = options['cx_model']['name']
if args.resume:
print('hello')
run_name = args.resume
save_dir = os.path.join(args.project_dir, 'logs', 'cx', run_name)
assert (os.path.isdir(save_dir))
i = 1
log_dir = os.path.join(args.project_dir, 'runs', run_name,
'resume_{}'.format(i))
while (os.path.isdir(log_dir)):
i += 1
log_dir = os.path.join(args.project_dir, 'runs', run_name,
'resume_{}'.format(i))
else:
print('why am i here')
run_name = datetime.now().strftime('%b%d-%H-%M-%S')
if args.comment:
run_name += '_' + args.comment
save_dir = os.path.join(args.project_dir, 'logs', 'cx', run_name)
if os.path.isdir(save_dir):
if click.confirm(
'Save directory already exists in {}. Erase?'.format(
save_dir)):
os.system('rm -r ' + save_dir)
else:
return
os.makedirs(os.path.join(save_dir, 'ckpt'))
os.makedirs(os.path.join(save_dir, 'best'))
# Tensorboard log directory
log_dir = os.path.join(args.project_dir, 'runs', run_name)
if args.viz:
viz_dir = os.path.join(args.project_dir, 'viz', 'cx', run_name)
print('viz_dir', viz_dir, run_name)
if os.path.isdir(viz_dir):
if click.confirm(
'Viz directory already exists in {}. Erase?'.format(
viz_dir)):
os.system('rm -r ' + viz_dir)
else:
return
os.makedirs(viz_dir)
train_writer = SummaryWriter(log_dir=os.path.join(log_dir, 'train'))
val_writer = SummaryWriter(log_dir=os.path.join(log_dir, 'val'))
print('Saving model to {}'.format(save_dir))
print('Logging results to {}'.format(log_dir))
##########################################################################
# Create datasets
##########################################################################
print('=> Loading VQA dataset...')
if args.dev_mode:
trainset_fname = 'trainset_augmented_small.pickle'
else:
trainset_fname = 'trainset_augmented.pickle'
trainset = pickle.load(
open(
os.path.join(options['vqa']['path_trainset'], 'pickle_old',
trainset_fname), 'rb'))
valset_fname = 'valset_augmented_small.pickle'
valset = pickle.load(
open(
os.path.join(options['vqa']['path_trainset'], 'pickle_old',
valset_fname), 'rb'))
if args.test:
testset_fname = 'valset_augmented.pickle'
testset = pickle.load(
open(
os.path.join(options['vqa']['path_trainset'], 'pickle_old',
testset_fname), 'rb'))
print('=> Loading KNN data...')
knns = json.load(open(options['coco']['path_knn'], 'r'))
knns = {int(k): v for k, v in knns.items()}
print('=> Loading COCO image features...')
features_train = h5py.File(
os.path.join(options['coco']['path_features'], 'trainset.hdf5'),
'r').get('noatt')
features_train = np.array(features_train)
# if not args.dev_mode:
features_val = h5py.File(
os.path.join(options['coco']['path_features'], 'valset.hdf5'),
'r').get('noatt')
features_val = np.array(features_val)
##########################################################################
# Create model
##########################################################################
print('=> Building model...')
vqa_model = None
optimizer = None
if args.cx_model == "RandomBaseline":
cx_model = RandomBaseline(knn_size=24)
elif args.cx_model == "DistanceBaseline":
cx_model = DistanceBaseline(knn_size=24)
else:
vqa_model = models.factory(options['model'],
trainset['vocab_words'],
trainset['vocab_answers'],
cuda=True,
data_parallel=True)
vqa_model = vqa_model.module
if args.pretrained_vqa:
load_vqa_checkpoint(
vqa_model, None,
os.path.join(options['logs']['dir_logs'], 'best'))
if args.cx_model == "BlackBox":
cx_model = BlackBox(vqa_model,
knn_size=24,
trainable_vqa=args.trainable_vqa)
elif args.cx_model == "LinearContext":
cx_model = LinearContext(vqa_model,
knn_size=24,
trainable_vqa=args.trainable_vqa)
elif args.cx_model == "SemanticBaseline":
if args.sb_lambda is None:
raise ValueError(
"If semantic baseline is selected then --sb_lambda must also be provided."
)
cx_model = SemanticBaseline(vqa_model,
knn_size=24,
trainable_vqa=args.trainable_vqa)
cx_model.set_lambda(args.sb_lambda)
emb = pickle.load(
open(
os.path.join(options['vqa']['path_trainset'],
"answer_embedding.pickle"), 'rb'))
cx_model.set_answer_embedding(emb)
elif args.cx_model == "NeuralModel":
model_spec = options['cx_model']
if model_spec['pretrained_emb']:
emb = pickle.load(
open(
os.path.join(options['vqa']['path_trainset'],
"answer_embedding.pickle"), 'rb'))
else:
emb = None
cx_model = NeuralModel(model_spec=model_spec,
dim_h=model_spec['dim_h'],
n_layers=model_spec['n_layers'],
emb=emb,
drop_p=model_spec['drop_p'],
vqa_model=vqa_model,
knn_size=24,
trainable_vqa=model_spec['trainable_vqa'])
elif args.cx_model == "PairwiseModel":
assert (args.pairwise)
cx_model = PairwiseModel(vqa_model,
knn_size=2,
trainable_vqa=args.trainable_vqa)
elif args.cx_model == "PairwiseLinearModel":
cx_model = PairwiseLinearModel(vqa_model,
knn_size=24,
trainable_vqa=args.trainable_vqa)
else:
raise ValueError("Unrecognized cx_model {}".format(args.cx_model))
optimizer = torch.optim.Adam(cx_model.parameters(),
lr=options['optim']['lr'])
print("Built {}".format(args.cx_model))
if args.resume:
info, start_epoch, best_recall = load_cx_checkpoint(
cx_model, save_dir, resume_best=args.best)
else:
info = []
start_epoch = 1
best_recall = 0
cx_model.cuda()
##########################################################################
# Train loop
##########################################################################
print('=> Starting training...')
if args.pairwise:
print('==> Pairwise training')
epoch = None
for epoch in range(start_epoch, options['optim']['epochs'] + 1):
cx_model.train()
if vqa_model is not None:
if options['cx_model']['trainable_vqa']:
vqa_model.train()
else:
vqa_model.eval()
train_b = 0
criterion = nn.CrossEntropyLoss(size_average=False)
trainset_batched = batchify(trainset['examples_list'],
batch_size=options['optim']['batch_size'])
for batch in tqdm(trainset_batched):
assert (cx_model.training)
if vqa_model is not None:
if options['cx_model']['trainable_vqa']:
assert (vqa_model.training)
else:
assert (not vqa_model.training)
image_features, question_wids, answer_aids, comp_idxs = getDataFromBatch(
batch,
features_train,
trainset['name_to_index'],
pairwise=args.pairwise)
scores = cx_model(image_features, question_wids, answer_aids)
if args.pairwise:
assert (scores.size(1) == 2)
zeros = Variable(torch.LongTensor([0] * len(batch))).cuda()
loss = criterion(scores, zeros) / len(batch)
correct = recallAtK(scores, zeros, k=1)
else:
correct = recallAtK(scores, comp_idxs, k=5)
loss = criterion(scores, comp_idxs) / len(batch)
if optimizer is not None:
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_b += 1
if train_b % args.print_freq == 0:
if args.pairwise:
metrics = {
'loss_pairwise': float(loss),
'acc_pairwise': (correct.sum() / len(batch))
}
else:
metrics = {
'loss': float(loss),
'recall': (correct.sum() / len(batch))
}
log_results(train_writer,
mode='train',
epoch=epoch,
i=((epoch - 1) * len(trainset_batched)) + train_b,
metrics=metrics)
if (args.eval_freq > 0 and train_b % args.eval_freq
== 0) or train_b == len(trainset_batched):
eval_results = eval_model(cx_model,
valset,
features_val,
options['optim']['batch_size'],
pairwise=args.pairwise)
log_results(val_writer,
mode='val',
epoch=epoch,
i=((epoch - 1) * len(trainset_batched)) + train_b,
metrics=eval_results)
info.append(eval_results)
if info[-1]['recall'] > best_recall:
is_best = True
best_recall = info[-1]['recall']
else:
is_best = False
save_cx_checkpoint(cx_model, info, save_dir, is_best=is_best)
if args.test or args.viz:
if epoch is not None:
_, best_epoch, _ = load_cx_checkpoint(cx_model,
save_dir,
resume_best=True)
else:
best_epoch = 0
if args.test:
test_results = eval_model(cx_model,
testset,
features_val,
options['optim']['batch_size'],
pairwise=args.pairwise)
test_results['best_epoch'] = best_epoch
with open(os.path.join(save_dir, 'final_results.txt'), 'w') as outfile:
outstr = json.dumps(test_results)
outfile.write(outstr)
print('Saved results to {}'.format(save_dir))
print('FINAL RESULTS ON BEST EPOCH {}'.format(best_epoch),
test_results)
if args.viz:
visualize_results(cx_model, valset, features_val, 200,
options['coco']['path_val_raw'], viz_dir)
def compute_prob_one_model(model_name, vqa_trainsplit="train"):
parser = argparse.ArgumentParser(
description='Train/Evaluate models',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
##################################################
# yaml options file contains all default choices #
# parser.add_argument('--path_opt', default='options/breast/default.yaml', type=str,
# help='path to a yaml options file')
parser.add_argument(
'--path_opt',
default='options/med/bilinear_att_train_imagenet_h200_g4.yaml',
type=str,
help='path to a yaml options file')
################################################
# change cli options to modify default choices #
# logs options
parser.add_argument(
'--dir_logs',
default='logs/med/train/globalbilinear_att_train_imagenet_h200_g4',
type=str,
help='dir logs')
# data options
parser.add_argument('--vqa_trainsplit',
type=str,
choices=['train', 'trainval'],
default=vqa_trainsplit)
# model options
parser.add_argument('--arch',
choices=models.model_names,
help='vqa model architecture: ' +
' | '.join(models.model_names))
parser.add_argument('--st_type', help='skipthoughts type')
parser.add_argument('--st_dropout', type=float)
parser.add_argument('--st_fixed_emb',
default=None,
type=utils.str2bool,
help='backprop on embedding')
# bert options
parser.add_argument(
'--bert_model',
default="bert-base-multilingual-uncased",
help=
'bert model: bert-base-uncased | bert-base-multilingual-uncased | bert-base-multilingual-cased'
)
# image options
parser.add_argument(
'--is_augment_image',
default='1',
help='whether to augment images at the beginning of every epoch?')
# optim options
parser.add_argument('-lr',
'--learning_rate',
type=float,
help='initial learning rate')
parser.add_argument('-b', '--batch_size', type=int, help='mini-batch size')
parser.add_argument('--epochs',
type=int,
help='number of total epochs to run')
# options not in yaml file
parser.add_argument('--start_epoch',
default=0,
type=int,
help='manual epoch number (useful on restarts)')
parser.add_argument('--resume',
default='best',
type=str,
help='path to latest checkpoint')
parser.add_argument('--save_model',
default=True,
type=utils.str2bool,
help='able or disable save model and optim state')
parser.add_argument(
'--save_all_from',
type=int,
help='''delete the preceding checkpoint until an epoch,'''
''' then keep all (useful to save disk space)')''')
parser.add_argument('-e',
'--evaluate',
dest='evaluate',
action='store_true',
help='evaluate model on validation and test set',
default=True)
parser.add_argument('-j',
'--workers',
default=0,
type=int,
help='number of data loading workers')
parser.add_argument('--print_freq',
'-p',
default=10,
type=int,
help='print frequency')
################################################
parser.add_argument('-ho',
'--help_opt',
dest='help_opt',
action='store_true',
help='show selected options before running')
args = parser.parse_args()
if vqa_trainsplit == "train":
args.dir_logs = "logs/med/train/{}".format(model_name)
else:
args.dir_logs = "logs/med/trainval/{}".format(model_name)
if "globalbilinear" in model_name:
path_opt = model_name.replace("globalbilinear", "bilinear")
if "_cased" in path_opt:
path_opt = path_opt.replace("_cased", "")
if "_uncased" in path_opt:
path_opt = path_opt.replace("_uncased", "")
elif "_cased_768" in model_name:
path_opt = model_name.replace("_cased_768", "_768")
elif "_uncased_768" in model_name:
path_opt = model_name.replace("_uncased_768", "_768")
elif "_cased" in model_name and "768" not in model_name:
path_opt = model_name.replace("_cased", "")
elif "_uncased" in model_name and "768" not in model_name:
path_opt = model_name.replace("_uncased", "")
else:
path_opt = model_name
path_opt = path_opt.replace("_trainval_", "_train_")
args.path_opt = "{}/{}.yaml".format(args.dir_logs, path_opt)
#########################################################################################
# Create options
#########################################################################################
if "_cased" in model_name:
args.bert_model = "bert-base-multilingual-cased"
elif "_uncased" in model_name:
args.bert_model = "bert-base-multilingual-uncased"
if args.bert_model == "bert-base-uncased":
question_features_path = BASE_EXTRACTED_QUES_FEATURES_PATH
elif args.bert_model == "bert-base-multilingual-cased":
question_features_path = CASED_EXTRACTED_QUES_FEATURES_PATH
else:
question_features_path = EXTRACTED_QUES_FEATURES_PATH
options = {
'vqa': {
'trainsplit': args.vqa_trainsplit
},
'logs': {
'dir_logs': args.dir_logs
},
'model': {
'arch': args.arch,
'seq2vec': {
'type': args.st_type,
'dropout': args.st_dropout,
'fixed_emb': args.st_fixed_emb
}
},
'optim': {
'lr': args.learning_rate,
'batch_size': args.batch_size,
'epochs': args.epochs
}
}
if args.path_opt is not None:
with open(args.path_opt, 'r') as handle:
options_yaml = yaml.load(handle, Loader=yaml.FullLoader)
options = utils.update_values(options, options_yaml)
print('## args')
pprint(vars(args))
print('## options')
pprint(options)
if args.help_opt:
return
# Set datasets options
if 'vgenome' not in options:
options['vgenome'] = None
#########################################################################################
# Create needed datasets
#########################################################################################
trainset = datasets.factory_VQA(options['vqa']['trainsplit'],
options['vqa'], options['coco'],
options['vgenome'])
train_loader = trainset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers,
shuffle=True)
if options['vqa']['trainsplit'] == 'train':
valset = datasets.factory_VQA('val', options['vqa'], options['coco'])
val_loader = valset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers)
if options['vqa']['trainsplit'] == 'trainval' or args.evaluate:
testset = datasets.factory_VQA('test', options['vqa'], options['coco'])
test_loader = testset.data_loader(
batch_size=options['optim']['batch_size'],
num_workers=args.workers)
#########################################################################################
# Create model, criterion and optimizer
#########################################################################################
model = models.factory(options['model'],
trainset.vocab_words(),
trainset.vocab_answers(),
cuda=True,
data_parallel=True)
criterion = criterions.factory(options['vqa'], cuda=True)
optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()),
options['optim']['lr'])
#########################################################################################
# args.resume: resume from a checkpoint OR create logs directory
#########################################################################################
exp_logger = None
if args.resume:
args.start_epoch, best_acc1, exp_logger = load_checkpoint(
model.module, optimizer,
os.path.join(options['logs']['dir_logs'], args.resume))
else:
# Or create logs directory
if os.path.isdir(options['logs']['dir_logs']):
if click.confirm(
'Logs directory already exists in {}. Erase?'.format(
options['logs']['dir_logs'], default=False)):
os.system('rm -r ' + options['logs']['dir_logs'])
else:
return
os.system('mkdir -p ' + options['logs']['dir_logs'])
path_new_opt = os.path.join(options['logs']['dir_logs'],
os.path.basename(args.path_opt))
path_args = os.path.join(options['logs']['dir_logs'], 'args.yaml')
with open(path_new_opt, 'w') as f:
yaml.dump(options, f, default_flow_style=False)
with open(path_args, 'w') as f:
yaml.dump(vars(args), f, default_flow_style=False)
if exp_logger is None:
# Set loggers
exp_name = os.path.basename(
options['logs']['dir_logs']) # add timestamp
exp_logger = logger.Experiment(exp_name, options)
exp_logger.add_meters('train', make_meters())
exp_logger.add_meters('test', make_meters())
if options['vqa']['trainsplit'] == 'train':
exp_logger.add_meters('val', make_meters())
exp_logger.info['model_params'] = utils.params_count(model)
print('Model has {} parameters'.format(
exp_logger.info['model_params']))
#########################################################################################
# args.evaluate: on valset OR/AND on testset
#########################################################################################
if args.evaluate:
path_logger_json = os.path.join(options['logs']['dir_logs'],
'logger.json')
if options['vqa']['trainsplit'] == 'train':
acc1, val_results, prob = engine.validate(
val_loader,
model,
criterion,
exp_logger,
args.start_epoch,
args.print_freq,
dict=io_utils.read_pickle(question_features_path),
bert_dim=options["model"]["dim_q"],
is_return_prob=True)
else:
test_results, testdev_results, prob = engine.test(
test_loader,
model,
exp_logger,
1,
args.print_freq,
dict=io_utils.read_pickle(question_features_path),
bert_dim=options["model"]["dim_q"],
is_return_prob=True)
torch.cuda.empty_cache()
if vqa_trainsplit == "train":
return prob, val_loader
else:
return prob, test_loader