def main():
log.info('[program starts.]')
checkpoint = torch.load(args.model)
opt = checkpoint['config']
state_dict = checkpoint['state_dict']
model = FusionNet_Model(opt, state_dict=state_dict)
log.info('[Model loaded.]')
test, test_embedding, test_ans = load_eval_data(opt, args.test_data)
model.setup_eval_embed(test_embedding)
log.info('[Data loaded.]')
if args.cuda:
model.cuda()
batches = BatchGen(test,
batch_size=args.batch_size,
evaluation=True,
gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
acc = sum([x == y
for x, y in zip(predictions, test_ans)]) / len(test_ans) * 100.0
print("Accuracy =", acc)
print(predictions[:args.show])
print(test_ans[:args.show])
def main():
log.info('[program starts.]')
checkpoint = torch.load(args.model)
opt = checkpoint['config']
opt['cuda'] = args.cuda
opt['seed'] = args.seed
opt['do_C2Q'] = True
state_dict = checkpoint['state_dict']
log.info('[model loaded.]')
test, test_embedding, questions, test_answer = load_test_data(opt)
model = LEGOReaderModel(opt, state_dict=state_dict)
log.info('[Data loaded.]')
model.setup_eval_embed(test_embedding)
if args.cuda:
model.cuda()
batches = BatchGen(test,
batch_size=args.batch_size,
evaluation=True,
gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, test_answer)
log.warning("Test EM: {} F1: {}".format(em, f1))
wrong_pred = []
with open(args.output, 'w', encoding='utf8') as f:
for i, pred in enumerate(predictions):
f.write(pred + '\n')
if _exact_match(pred, test_answer[i]) == False:
wrong_pred.append((i, pred))
error_samples = random.sample(wrong_pred, args.show)
for (i, pred) in error_samples:
print('Context: ', test[i][-2])
print('Question: ', questions[i], '\n')
print('Answers: ', test_answer[i])
print('Predictions: ', pred)
print('\n==================\n')
def main():
log.info('[program starts.]')
opt = vars(args) # changing opt will change args
train, train_embedding, opt = load_train_data(opt)
dev, dev_embedding, dev_answer = load_dev_data(opt)
log.info('[Data loaded.]')
if args.resume:
log.info('[loading previous model...]')
checkpoint = torch.load(args.resume)
if args.resume_options:
opt = checkpoint['config']
state_dict = checkpoint['state_dict']
model = LEGOReaderModel(opt, train_embedding, state_dict)
epoch_0 = checkpoint['epoch'] + 1
for i in range(checkpoint['epoch']):
random.shuffle(list(range(len(train)))) # synchronize random seed
if args.reduce_lr:
lr_decay(model.optimizer, lr_decay=args.reduce_lr)
else:
model = LEGOReaderModel(opt, train_embedding)
epoch_0 = 1
model.setup_eval_embed(dev_embedding)
log.info("[dev] Total number of params: {}".format(model.total_param))
if args.cuda:
model.cuda()
if args.resume:
batches = BatchGen(dev,
batch_size=args.batch_size,
evaluation=True,
gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_answer)
log.info("[dev EM: {} F1: {}]".format(em, f1))
best_val_score = f1
else:
best_val_score = 0.0
for epoch in range(epoch_0, epoch_0 + args.epoches):
log.warning('Epoch {}'.format(epoch))
# train
batches = BatchGen(train, batch_size=args.batch_size, gpu=args.cuda)
start = datetime.now()
for i, batch in enumerate(batches):
model.update(batch)
if i % args.log_per_updates == 0:
log.info(
'updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'.format(
model.updates, model.train_loss.avg,
str((datetime.now() - start) / (i + 1) *
(len(batches) - i - 1)).split('.')[0]))
# eval
if epoch % args.eval_per_epoch == 0:
batches = BatchGen(dev,
batch_size=args.batch_size,
evaluation=True,
gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
em, f1 = score(predictions, dev_answer)
log.warning("Epoch {} - dev EM: {} F1: {}".format(epoch, em, f1))
# save
if args.save_best_only:
if f1 > best_val_score:
best_val_score = f1
model_file = os.path.join(model_dir, 'best_model.pt')
model.save(model_file, epoch)
log.info('[new best model saved.]')
else:
model_file = os.path.join(model_dir,
'checkpoint_epoch_{}.pt'.format(epoch))
model.save(model_file, epoch)
if f1 > best_val_score:
best_val_score = f1
copyfile(os.path.join(model_dir, model_file),
os.path.join(model_dir, 'best_model.pt'))
log.info('[new best model saved.]')
def main():
log.info('[program starts.]')
opt = vars(args) # changing opt will change args
train, train_embedding, opt = load_train_data(opt, args.train_meta,
args.train_data)
dev, dev_embedding, dev_ans = load_eval_data(opt, args.dev_data)
test, test_embedding, test_ans = load_eval_data(opt, args.test_data)
log.info('[Data loaded.]')
model = FusionNet_Model(opt, train_embedding)
if args.cuda: model.cuda()
log.info("[dev] Total number of params: {}".format(model.total_param))
best_acc = 0.0
for epoch in range(1, 1 + args.epoches):
log.warning('Epoch {}'.format(epoch))
# train
batches = BatchGen(train, batch_size=args.batch_size, gpu=args.cuda)
start = datetime.now()
for i, batch in enumerate(batches):
model.update(batch)
if i % args.log_per_updates == 0:
log.info(
'updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'.format(
model.updates, model.train_loss.avg,
str((datetime.now() - start) / (i + 1) *
(len(batches) - i - 1)).split('.')[0]))
# dev eval
model.setup_eval_embed(dev_embedding)
if args.cuda: model.cuda()
batches = BatchGen(dev,
batch_size=args.batch_size,
evaluation=True,
gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
acc = sum([x == y for x, y in zip(predictions, dev_ans)
]) / len(dev_ans) * 100.0
# test (or dev 2) eval
model.setup_eval_embed(test_embedding)
if args.cuda: model.cuda()
batches = BatchGen(test,
batch_size=args.batch_size,
evaluation=True,
gpu=args.cuda)
predictions = []
for batch in batches:
predictions.extend(model.predict(batch))
corr_acc = sum([x == y for x, y in zip(predictions, test_ans)
]) / len(test_ans) * 100.0
# save for predict
if args.do_not_save == False:
if args.save_best_only:
if (acc + corr_acc) / 2 > best_acc:
model_file = os.path.join(model_dir, 'best_model.pt')
model.save_for_predict(model_file, epoch)
log.info('[new best model saved.]')
else:
model_file = os.path.join(
model_dir, 'checkpoint_epoch_{}.pt'.format(epoch))
model.save_for_predict(model_file, epoch)
if (acc + corr_acc) / 2 > best_acc:
copyfile(os.path.join(model_dir, model_file),
os.path.join(model_dir, 'best_model.pt'))
log.info('[new best model saved.]')
if (acc + corr_acc) / 2 > best_acc:
best_acc = (acc + corr_acc) / 2
log.warning(
"Epoch {0} - dev Acc: {1:.3f}, dev2 Acc: {2:.3f} (best Acc: {3:.3f})"
.format(epoch, acc, corr_acc, best_acc))