def train_epoch(data_loader, model, logger, predict_during_train=False, epoch=1):
model.train()
pbar = tqdm(total=len(data_loader))
epoch_len = len(data_loader)
step_count = 0
predict_step = epoch_len // 2
while not data_loader.empty():
step_count += 1
batch = next(iter(data_loader))
batch['context_mask'] = batch['context_mask'].float()
train_batch(model, batch)
# del batch
if predict_during_train and (step_count % predict_step == 0):
predict(model, eval_dataset, dev_example_dict, dev_feature_dict,
join(args.prediction_path, 'pred_seed_{}_epoch_{}_{}.json'.format(args.seed, epc, step_count)))
eval(join(args.prediction_path, 'pred_seed_{}_epoch_{}_{}.json'.format(args.seed, epc, step_count)), args.validdata)
model_to_save = model.module if hasattr(model, 'module') else model
torch.save(model_to_save.state_dict(), join(args.checkpoint_path, "ckpt_seed_{}_epoch_{}_{}.pkl".format(args.seed, epc, step_count)))
model.train()
pbar.update(1)
predict(model, eval_dataset, dev_example_dict, dev_feature_dict,
join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epc)))
results = eval(join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epc)), args.validdata)
# Logging
keys='em,f1,prec,recall,sp_em,sp_f1,sp_prec,sp_recall,joint_em,joint_f1,joint_prec,joint_recall'.split(',')
logger.info(','.join([str(epoch)] + [str(results[s]) for s in keys]))
model_to_save = model.module if hasattr(model, 'module') else model
torch.save(model_to_save.state_dict(), join(args.checkpoint_path, "ckpt_seed_{}_epoch_{}_99999.pkl".format(args.seed, epc)))
def train_epoch(data_loader,eval_dataset, dev_example_dict, dev_feature_dict, model, optimizer, scheduler, criterion,sp_loss_fct, logger, predict_during_train=False, epoch=1, global_step=0, test_loss_record=None):
model.train()
pbar = tqdm(total=len(data_loader))
epoch_len = len(data_loader)
step_count = 0
predict_step = epoch_len // 2
for x, batch in enumerate(data_loader):
step_count += 1
# batch = next(iter(data_loader))
batch['context_mask'] = batch['context_mask'].float()
train_batch(model, optimizer, scheduler,criterion,sp_loss_fct, batch, global_step)
global_step+=1
# del batch
if predict_during_train and (step_count % predict_step == 0):
predict(model, eval_dataset, dev_example_dict, dev_feature_dict,
join(args.prediction_path, 'pred_seed_{}_epoch_{}_{}.json'.format(args.seed, epoch, step_count)))
eval(join(args.prediction_path, 'pred_seed_{}_epoch_{}_{}.json'.format(args.seed, epoch, step_count)), args.validdata)
model_to_save = model.module if hasattr(model, 'module') else model
torch.save(model_to_save.state_dict(), join(args.checkpoint_path, "ckpt_seed_{}_epoch_{}_{}.pkl".format(args.seed, epoch, step_count)))
model.train()
pbar.update(1)
#only save model in MASTER core.
if xm.get_ordinal() == 0:
predict(model, eval_dataset, dev_example_dict, dev_feature_dict,
join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epoch)),
test_loss_record)
results = eval(join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epoch)), args.validdata)
# Logging
keys='em,f1,prec,recall,sp_em,sp_f1,sp_prec,sp_recall,joint_em,joint_f1,joint_prec,joint_recall'.split(',')
logger.info(','.join([str(epoch)] + [str(results[s]) for s in keys]))
model_to_save = model.module if hasattr(model, 'module') else model
torch.save(model_to_save.state_dict(), join(args.checkpoint_path, "model_{}.bin".format(epoch)))
def _epoch_evaluate_update_description_log(self, tqdm_obj, logger, epoch,
exam, feats, gold_file):
"""Evaluate model and update logs for epoch.
Args:
tqdm_obj: tqdm/trange object with description to be updated
logger: logging.logger
epoch: int
Return:
train_acc, train_f1, valid_acc, valid_f1
"""
# Evaluate model for train and valid set
# results = self._evaluate_for_train_valid()
# train_acc, train_f1, valid_acc, valid_f1 = results
# step_count = 999
self.predict(
self.model, tqdm_obj, exam, feats,
join(self.config.prediction_path,
'pred_epoch_{}.json'.format(epoch)))
results = eval(
join(self.config.prediction_path,
'pred_epoch_{}.json'.format(epoch)), gold_file)
# Update tqdm description for command line
# tqdm_obj.set_description(
# 'Epoch: {:d}, train_acc: {:.6f}, train_f1: {:.6f}, '
# 'valid_acc: {:.6f}, valid_f1: {:.6f}, '.format(
# epoch, train_acc, train_f1, valid_acc, valid_f1))
# # Logging
# logger.info(','.join([str(epoch)] + [str(s) for s in results]))
return results
def test_fn(data_loader, dev_example_dict, dev_feature_dict, model, optimizer, scheduler,
criterion, sp_loss_fct, logger, predict_during_train=False, epoch=1, global_step=0,
test_loss_record=None):
model.train()
# pbar = tqdm(total=len(eval_dataset))
epoch_len = len(data_loader)
step_count = 0
predict_step = epoch_len // 2
# for x, batch in enumerate(data_loader):
predict(model, data_loader, dev_example_dict, dev_feature_dict,
join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epoch)),
test_loss_record)
results = eval(join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epoch)),
args.validdata)
# Logging
keys = 'em,f1,prec,recall,sp_em,sp_f1,sp_prec,sp_recall,joint_em,joint_f1,joint_prec,joint_recall'.split(
',')
logger.info(','.join([str(epoch)] + [str(results[s]) for s in keys]))