def train_epoch(data_loader, model, logger, predict_during_train=False, epoch=1): model.train() pbar = tqdm(total=len(data_loader)) epoch_len = len(data_loader) step_count = 0 predict_step = epoch_len // 2 while not data_loader.empty(): step_count += 1 batch = next(iter(data_loader)) batch['context_mask'] = batch['context_mask'].float() train_batch(model, batch) # del batch if predict_during_train and (step_count % predict_step == 0): predict(model, eval_dataset, dev_example_dict, dev_feature_dict, join(args.prediction_path, 'pred_seed_{}_epoch_{}_{}.json'.format(args.seed, epc, step_count))) eval(join(args.prediction_path, 'pred_seed_{}_epoch_{}_{}.json'.format(args.seed, epc, step_count)), args.validdata) model_to_save = model.module if hasattr(model, 'module') else model torch.save(model_to_save.state_dict(), join(args.checkpoint_path, "ckpt_seed_{}_epoch_{}_{}.pkl".format(args.seed, epc, step_count))) model.train() pbar.update(1) predict(model, eval_dataset, dev_example_dict, dev_feature_dict, join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epc))) results = eval(join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epc)), args.validdata) # Logging keys='em,f1,prec,recall,sp_em,sp_f1,sp_prec,sp_recall,joint_em,joint_f1,joint_prec,joint_recall'.split(',') logger.info(','.join([str(epoch)] + [str(results[s]) for s in keys])) model_to_save = model.module if hasattr(model, 'module') else model torch.save(model_to_save.state_dict(), join(args.checkpoint_path, "ckpt_seed_{}_epoch_{}_99999.pkl".format(args.seed, epc)))
def train_epoch(data_loader,eval_dataset, dev_example_dict, dev_feature_dict, model, optimizer, scheduler, criterion,sp_loss_fct, logger, predict_during_train=False, epoch=1, global_step=0, test_loss_record=None): model.train() pbar = tqdm(total=len(data_loader)) epoch_len = len(data_loader) step_count = 0 predict_step = epoch_len // 2 for x, batch in enumerate(data_loader): step_count += 1 # batch = next(iter(data_loader)) batch['context_mask'] = batch['context_mask'].float() train_batch(model, optimizer, scheduler,criterion,sp_loss_fct, batch, global_step) global_step+=1 # del batch if predict_during_train and (step_count % predict_step == 0): predict(model, eval_dataset, dev_example_dict, dev_feature_dict, join(args.prediction_path, 'pred_seed_{}_epoch_{}_{}.json'.format(args.seed, epoch, step_count))) eval(join(args.prediction_path, 'pred_seed_{}_epoch_{}_{}.json'.format(args.seed, epoch, step_count)), args.validdata) model_to_save = model.module if hasattr(model, 'module') else model torch.save(model_to_save.state_dict(), join(args.checkpoint_path, "ckpt_seed_{}_epoch_{}_{}.pkl".format(args.seed, epoch, step_count))) model.train() pbar.update(1) #only save model in MASTER core. if xm.get_ordinal() == 0: predict(model, eval_dataset, dev_example_dict, dev_feature_dict, join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epoch)), test_loss_record) results = eval(join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epoch)), args.validdata) # Logging keys='em,f1,prec,recall,sp_em,sp_f1,sp_prec,sp_recall,joint_em,joint_f1,joint_prec,joint_recall'.split(',') logger.info(','.join([str(epoch)] + [str(results[s]) for s in keys])) model_to_save = model.module if hasattr(model, 'module') else model torch.save(model_to_save.state_dict(), join(args.checkpoint_path, "model_{}.bin".format(epoch)))
def _epoch_evaluate_update_description_log(self, tqdm_obj, logger, epoch, exam, feats, gold_file): """Evaluate model and update logs for epoch. Args: tqdm_obj: tqdm/trange object with description to be updated logger: logging.logger epoch: int Return: train_acc, train_f1, valid_acc, valid_f1 """ # Evaluate model for train and valid set # results = self._evaluate_for_train_valid() # train_acc, train_f1, valid_acc, valid_f1 = results # step_count = 999 self.predict( self.model, tqdm_obj, exam, feats, join(self.config.prediction_path, 'pred_epoch_{}.json'.format(epoch))) results = eval( join(self.config.prediction_path, 'pred_epoch_{}.json'.format(epoch)), gold_file) # Update tqdm description for command line # tqdm_obj.set_description( # 'Epoch: {:d}, train_acc: {:.6f}, train_f1: {:.6f}, ' # 'valid_acc: {:.6f}, valid_f1: {:.6f}, '.format( # epoch, train_acc, train_f1, valid_acc, valid_f1)) # # Logging # logger.info(','.join([str(epoch)] + [str(s) for s in results])) return results
def test_fn(data_loader, dev_example_dict, dev_feature_dict, model, optimizer, scheduler, criterion, sp_loss_fct, logger, predict_during_train=False, epoch=1, global_step=0, test_loss_record=None): model.train() # pbar = tqdm(total=len(eval_dataset)) epoch_len = len(data_loader) step_count = 0 predict_step = epoch_len // 2 # for x, batch in enumerate(data_loader): predict(model, data_loader, dev_example_dict, dev_feature_dict, join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epoch)), test_loss_record) results = eval(join(args.prediction_path, 'pred_seed_{}_epoch_{}_99999.json'.format(args.seed, epoch)), args.validdata) # Logging keys = 'em,f1,prec,recall,sp_em,sp_f1,sp_prec,sp_recall,joint_em,joint_f1,joint_prec,joint_recall'.split( ',') logger.info(','.join([str(epoch)] + [str(results[s]) for s in keys]))