def train_epochs(args, model, optimizer, params, dicts): """ Main loop. does train and test """ metrics_hist = defaultdict(lambda: []) metrics_hist_te = defaultdict(lambda: []) metrics_hist_tr = defaultdict(lambda: []) test_only = args.test_model is not None #train for n_epochs unless criterion metric does not improve for [patience] epochs for epoch in range(args.n_epochs): #only test on train/test set on very last epoch if epoch == 0 and not args.test_model: model_dir = os.path.join( MODEL_DIR, '_'.join( [args.model, time.strftime('%b_%d_%H:%M', time.gmtime())])) os.mkdir(model_dir) elif args.test_model: model_dir = os.getcwd( ) #just save things to where this script was called metrics_all = one_epoch(model, optimizer, epoch, args.n_epochs, args.batch_size, args.data_path, test_only, dicts, model_dir, args.samples, args.gpu, args.debug, args.quiet) # DISTRIBUTING results from metrics_all to respective dicts for name in metrics_all[0].keys(): metrics_hist[name].append(metrics_all[0][name]) for name in metrics_all[1].keys(): metrics_hist_te[name].append(metrics_all[1][name]) for name in metrics_all[2].keys(): metrics_hist_tr[name].append(metrics_all[2][name]) metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr) #save metrics, model, params persistence.save_everything( args, metrics_hist_all, model, model_dir, params, args.criterion ) # SHOULD SAVE MODEL PARAMS AT EACH EPOCH, BELIEVE IS HAPPENING if test_only: break if args.criterion in metrics_hist.keys(): if early_stop(metrics_hist, args.criterion, args.patience): #stop training, do tests on test and train sets, and then stop the script print("%s hasn't improved in %d epochs, early stopping..." % (args.criterion, args.patience)) test_only = True model = torch.load( '%s/model_best_%s.pth' % (model_dir, args.criterion)) # LOADING BEST MODEL FOR FINAL TEST return epoch + 1
def train_epochs(args, model, optimizer, params, dicts): """ Main loop. does train and test """ metrics_hist = defaultdict(lambda: []) metrics_hist_te = defaultdict(lambda: []) metrics_hist_tr = defaultdict(lambda: []) test_only = args.test_model is not None evaluate = args.test_model is not None #train for n_epochs unless criterion metric does not improve for [patience] epochs for epoch in range(args.n_epochs): #only test on train/test set on very last epoch if epoch == 0 and not args.test_model: model_dir = os.path.join( MODEL_DIR, '_'.join([ args.model, time.strftime('%b_%d_%H:%M:%S', time.localtime()) ])) os.mkdir(model_dir) elif args.test_model: model_dir = os.path.dirname(os.path.abspath(args.test_model)) metrics_all = one_epoch(model, optimizer, args.Y, epoch, args.n_epochs, args.batch_size, args.data_path, args.version, test_only, dicts, model_dir, args.samples, args.gpu, args.quiet) for name in metrics_all[0].keys(): metrics_hist[name].append(metrics_all[0][name]) for name in metrics_all[1].keys(): metrics_hist_te[name].append(metrics_all[1][name]) for name in metrics_all[2].keys(): metrics_hist_tr[name].append(metrics_all[2][name]) metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr) #save metrics, model, params persistence.save_everything(args, metrics_hist_all, model, model_dir, params, args.criterion, evaluate) if test_only: #we're done break if args.criterion in metrics_hist.keys(): if early_stop(metrics_hist, args.criterion, args.patience): #stop training, do tests on test and train sets, and then stop the script print("%s hasn't improved in %d epochs, early stopping..." % (args.criterion, args.patience)) test_only = True args.test_model = '%s/model_best_%s.pth' % (model_dir, args.criterion) model = tools.pick_model(args, dicts) return epoch + 1
def train_epochs(args, model, optimizer, params, freq_params, dicts): """ Main loop. does train and test """ metrics_hist = defaultdict(lambda: []) metrics_hist_te = defaultdict(lambda: []) metrics_hist_tr = defaultdict(lambda: []) test_only = args.test_model is not None unseen_code_inds = set() #train for n_epochs unless criterion metric does not improve for [patience] epochs for epoch in range(args.n_epochs): #only test on train/test set on very last epoch if epoch == 0 and not args.test_model: model_dir = os.path.join( MODEL_DIR, '_'.join( [args.model, time.strftime('%b_%d_%H:%M', time.gmtime())])) os.mkdir(model_dir) elif args.test_model: #just save things to where this script was called model_dir = os.getcwd() metrics_all, unseen_code_inds = one_epoch( model, optimizer, args.Y, epoch, args.n_epochs, args.batch_size, args.data_path, args.version, freq_params, test_only, dicts, model_dir, unseen_code_inds, args.samples, args.gpu, args.debug, args.quiet) for name in metrics_all[0].keys(): metrics_hist[name].append(metrics_all[0][name]) for name in metrics_all[1].keys(): metrics_hist_te[name].append(metrics_all[1][name]) for name in metrics_all[2].keys(): metrics_hist_tr[name].append(metrics_all[2][name]) metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr) #save metrics, model, params persistence.save_everything(args, metrics_hist_all, model, model_dir, params, args.criterion) if test_only: break if args.criterion in metrics_hist.keys(): if early_stop(metrics_hist, args.criterion, args.patience): #stop training, do tests on test and train sets, and then stop the script print("%s hasn't improved in %d epochs, early stopping..." % (args.criterion, args.patience)) test_only = True model = torch.load('%s/model_best_%s.pth' % (model_dir, args.criterion)) return epoch + 1
def train_epochs(args, model, optimizer, params, dicts, start_epoch): """ Main loop. does train and test """ metrics_hist = defaultdict(lambda: []) metrics_hist_te = defaultdict(lambda: []) metrics_hist_tr = defaultdict(lambda: []) if args.reload_model: #get existing metrics up to model checkpoint: with open(os.path.join(args.reload_model, 'metrics.json'), 'r') as f: metrics_hist_all = json.load(f) #this will be a little expensive-- iterate through and re-assign: for el in metrics_hist_all: if '_te' in el: metrics_hist_te[re.sub( '_te$', '', el)] = metrics_hist_all[el][:start_epoch] elif '_tr' in el: metrics_hist_tr[re.sub( '_tr$', '', el)] = metrics_hist_all[el][:start_epoch] else: #dev metrics_hist[el] = metrics_hist_all[el][:start_epoch] META_TEST = args.test_model is not None test_only = args.test_model is not None evaluate = args.test_model is not None #train for n_epochs unless criterion metric does not improve for [patience] epochs for epoch in range( start_epoch, args.n_epochs ): #only train for _x_ more epochs from best-saved model* #only test on train/test set on very last epoch if epoch == start_epoch and not args.test_model: model_dir = os.path.join( MODEL_DIR, '_'.join([ args.model, time.strftime('%b_%d_%H:%M', time.localtime()) ])) os.mkdir(model_dir) #save model versioning (git) info: repo = git.Repo(search_parent_directories=True) branch = repo.active_branch.name print("branch:", branch) sha = repo.head.object.hexsha print("SHA hash:", sha) persistence.save_git_versioning_info( model_dir, (branch, sha, args.description)) elif args.test_model: model_dir = os.path.dirname(os.path.abspath(args.test_model)) metrics_all = one_epoch(args, model, optimizer, epoch, test_only, dicts, model_dir, META_TEST) for name in metrics_all[0].keys(): metrics_hist[name].append(metrics_all[0][name]) for name in metrics_all[1].keys(): metrics_hist_te[name].append(metrics_all[1][name]) for name in metrics_all[2].keys(): metrics_hist_tr[name].append(metrics_all[2][name]) metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr) #save metrics, model, params assert '-' not in args.criterion persistence.save_everything(args, metrics_hist_all, model, model_dir, params, optimizer, evaluate) if test_only: #we're done break if args.criterion in metrics_hist.keys(): if early_stop(metrics_hist, args.criterion, args.patience): #stop training, do tests on test and train sets, and then stop the script print("%s hasn't improved in %d epochs, early stopping..." % (args.criterion, args.patience)) test_only = True test_m = [ o for o in os.listdir(model_dir) if 'model_best' in o ] assert (len(test_m) == 1) args.test_model = os.path.join(model_dir, test_m[0]) model, _, _ = tools.pick_model(args, dicts, META_TEST) return epoch - start_epoch + 1
def train_epochs(args, model, optimizer, params, dicts): """ Main loop. does train and test """ metrics_hist_train = defaultdict(list) metrics_hist_dev = defaultdict(list) metrics_hist_test = defaultdict(list) if args.resume: metrics_file = os.path.join( os.path.dirname(os.path.abspath(args.resume)), 'metrics.json') if os.path.exists(metrics_file): with open(metrics_file, 'r') as metrics_fp: metrics = json.load(metrics_fp) metrics_hist_train.update({ k.replace('_train', ''): v for k, v in metrics.items() if k.endswith('_train') }) metrics_hist_dev.update({ k.replace('_dev', ''): v for k, v in metrics.items() if k.endswith('_dev') }) metrics_hist_test.update({ k.replace('_test', ''): v for k, v in metrics.items() if k.endswith('_test') }) test_only = args.test_model is not None num_labels_fine = len(dicts['ind2c']) num_labels_coarse = len(dicts['ind2c_coarse']) epoch = 0 if args.resume is None else args.epoch if not test_only: dataset_train = MimicDataset(args.data_path, dicts, num_labels_fine, num_labels_coarse, args.max_len) dataset_dev = MimicDataset(args.data_path.replace('train', 'dev'), dicts, num_labels_fine, num_labels_coarse, args.max_len) if args.resume is None: model_dir = os.path.join( args.models_dir, '_'.join([args.model, time.strftime('%Y-%m-%d_%H:%M:%S')])) os.mkdir(model_dir) else: model_dir = os.path.dirname(os.path.abspath(args.resume)) else: model_dir = os.path.dirname(os.path.abspath(args.test_model)) #tensorboard = Tensorboard(model_dir) #train for n_epochs unless criterion metric does not improve for [patience] epochs for epoch in range(epoch, args.n_epochs if not test_only else 0): losses = train(model, optimizer, args.Y, epoch, args.batch_size, args.embed_desc, dataset_train, args.shuffle, args.gpu, dicts) loss = np.mean(losses) metrics_train = {'loss': loss} fold = 'dev' #evaluate on dev with torch.no_grad(): metrics_dev, _, _, _ = test(model, args.Y, epoch, dataset_dev, args.batch_size, args.embed_desc, fold, args.gpu, dicts, model_dir) for name, val in metrics_train.items(): #tensorboard.log_scalar('%s_train' % (name), val, epoch) metrics_hist_train[name].append(metrics_train[name]) metrics_hist_train.update({'epochs': epoch + 1}) for name, val in metrics_dev.items(): #tensorboard.log_scalar('%s_dev' % (name), val, epoch) metrics_hist_dev[name].append(metrics_dev[name]) metrics_hist_all = (metrics_hist_train, metrics_hist_dev, None) #save metrics, model, optimizer state, params persistence.save_everything(args, dicts, metrics_hist_all, model, optimizer, model_dir, params, args.criterion, evaluate=False, test_only=False) if args.criterion is not None: if early_stop(metrics_hist_dev, args.criterion, args.patience): #stop training, evaluate on test set and then stop the script print('{} hasn\'t improved in {} epochs, early stopping...'. format(args.criterion, args.patience)) break fold = 'test' print("\nevaluating on test") dataset_train = None dataset_dev = None del dataset_train, dataset_dev if not test_only: model_best_sd = torch.load( os.path.join(model_dir, 'model_best_{}.pth'.format(args.criterion))) model.load_state_dict(model_best_sd) if args.gpu: model.cuda() dataset_test = MimicDataset(args.data_path.replace('train', 'test'), dicts, num_labels_fine, num_labels_coarse, args.max_len) with torch.no_grad(): metrics_test, metrics_codes, metrics_inst, hadm_ids = test( model, args.Y, epoch, dataset_test, args.batch_size, args.embed_desc, fold, args.gpu, dicts, model_dir) for name, val in metrics_test.items(): #if not test_only: # tensorboard.log_scalar('%s_test' % (name), val, epoch) metrics_hist_test[name].append(metrics_test[name]) metrics_hist_all = (metrics_hist_train, metrics_hist_dev, metrics_hist_test) #tensorboard.close() #save metrics, model, params persistence.save_everything(args, dicts, metrics_hist_all, model, optimizer, model_dir, params, args.criterion, metrics_codes=metrics_codes, metrics_inst=metrics_inst, hadm_ids=hadm_ids, evaluate=True, test_only=test_only) return epoch + 1, metrics_hist_test
def train_epochs(args, model, optimizer, params, dicts, scheduler, labels_weight): """ Main loop. does train and test """ dev_acc_macro, dev_prec_macro, dev_rec_macro, dev_f1_macro = 0.0, 0.0, 0.0, 0.0 dev_acc_micro, dev_prec_micro, dev_rec_micro, dev_f1_micro = 0.0, 0.0, 0.0, 0.0 dev_rec_at_5, dev_prec_at_5, dev_f1_at_5 = 0.0, 0.0, 0.0 dev_auc_macro, dev_auc_micro = 0.0, 0.0 dev_epoch, dev_loss = 0.0, 0.0 test_acc_macro, test_prec_macro, test_rec_macro, test_f1_macro = 0.0, 0.0, 0.0, 0.0 test_acc_micro, test_prec_micro, test_rec_micro, test_f1_micro = 0.0, 0.0, 0.0, 0.0 test_rec_at_5, test_prec_at_5, test_f1_at_5 = 0.0, 0.0, 0.0 test_auc_macro, test_auc_micro = 0.0, 0.0 test_epoch, test_loss = 0.0, 0.0 metrics_hist = defaultdict(lambda: []) metrics_hist_te = defaultdict(lambda: []) metrics_hist_tr = defaultdict(lambda: []) test_only = args.test_model is not None evaluate = args.test_model is not None # train for n_epochs unless criterion metric does not improve for [patience] epochs for epoch in range(args.n_epochs): best = False #only test on train/test set on very last epoch if epoch == 0 and not args.test_model: model_dir = os.path.join( MODEL_DIR, '_'.join([ args.model, time.strftime('%b_%d_%H:%M:%S', time.localtime()) ])) os.makedirs(model_dir) elif args.test_model: model_dir = os.path.dirname(os.path.abspath(args.test_model)) metrics_all = one_epoch(args, \ model, \ optimizer, \ args.Y, \ epoch, \ args.n_epochs, \ args.batch_size, \ args.data_path, \ args.version, \ test_only, \ dicts, \ model_dir, \ args.samples, \ args.gpu, \ args.quiet, \ scheduler, \ labels_weight, \ ) for name in metrics_all[0].keys(): metrics_hist[name].append(metrics_all[0][name]) for name in metrics_all[1].keys(): metrics_hist_te[name].append(metrics_all[1][name]) for name in metrics_all[2].keys(): metrics_hist_tr[name].append(metrics_all[2][name]) metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr) if metrics_hist_te['auc_micro'][-1] >= test_auc_micro: test_acc_macro = metrics_hist_te['acc_macro'][-1] test_prec_macro = metrics_hist_te['prec_macro'][-1] test_rec_macro = metrics_hist_te['rec_macro'][-1] test_f1_macro = metrics_hist_te['f1_macro'][-1] test_acc_micro = metrics_hist_te['acc_micro'][-1] test_prec_micro = metrics_hist_te['prec_micro'][-1] test_rec_micro = metrics_hist_te['rec_micro'][-1] test_f1_micro = metrics_hist_te['f1_micro'][-1] if not args.Y == 'full': test_rec_at_5 = metrics_hist_te['rec_at_5'][-1] test_prec_at_5 = metrics_hist_te['prec_at_5'][-1] test_f1_at_5 = metrics_hist_te['f1_at_5'][-1] test_auc_macro = metrics_hist_te['auc_macro'][-1] test_auc_micro = metrics_hist_te['auc_micro'][-1] test_loss = metrics_hist_te['loss_test'][-1] test_epoch = epoch if metrics_hist['auc_micro'][-1] >= dev_auc_micro: dev_acc_macro = metrics_hist['acc_macro'][-1] dev_prec_macro = metrics_hist['prec_macro'][-1] dev_rec_macro = metrics_hist['rec_macro'][-1] dev_f1_macro = metrics_hist['f1_macro'][-1] dev_acc_micro = metrics_hist['acc_micro'][-1] dev_prec_micro = metrics_hist['prec_micro'][-1] dev_rec_micro = metrics_hist['rec_micro'][-1] dev_f1_micro = metrics_hist['f1_micro'][-1] if not args.Y == 'full': dev_rec_at_5 = metrics_hist['rec_at_5'][-1] dev_prec_at_5 = metrics_hist['prec_at_5'][-1] dev_f1_at_5 = metrics_hist['f1_at_5'][-1] dev_auc_macro = metrics_hist['auc_macro'][-1] dev_auc_micro = metrics_hist['auc_micro'][-1] dev_loss = metrics_hist['loss_dev'][-1] dev_epoch = epoch best = True print() print('-' * 19 + ' Best (Dev) ' + '-' * 19) print('Best Epoch: %d' % dev_epoch) print() print("[MACRO] accuracy, precision, recall, f-measure, AUC") print("%.4f, %.4f, %.4f, %.4f, %.4f" % (dev_acc_macro, dev_prec_macro, dev_rec_macro, dev_f1_macro, dev_auc_macro)) print("[MICRO] accuracy, precision, recall, f-measure, AUC") print("%.4f, %.4f, %.4f, %.4f, %.4f" % (dev_acc_micro, dev_prec_micro, dev_rec_micro, dev_f1_micro, dev_auc_micro)) print('rec_at_5: %.4f' % dev_rec_at_5) print('prec_at_5: %.4f' % dev_prec_at_5) print() print('Dev loss: %.4f' % dev_loss) print() print('-' * 51) print() print('-' * 19 + ' Best (Test) ' + '-' * 19) print('Best Epoch: %d' % test_epoch) print() print("[MACRO] accuracy, precision, recall, f-measure, AUC") print("%.4f, %.4f, %.4f, %.4f, %.4f" % (test_acc_macro, test_prec_macro, test_rec_macro, test_f1_macro, test_auc_macro)) print("[MICRO] accuracy, precision, recall, f-measure, AUC") print("%.4f, %.4f, %.4f, %.4f, %.4f" % (test_acc_micro, test_prec_micro, test_rec_micro, test_f1_micro, test_auc_micro)) print('rec_at_5: %.4f' % test_rec_at_5) print('prec_at_5: %.4f' % test_prec_at_5) print() print('Test loss: %.4f' % test_loss) print() print('-' * 51) # save metrics, model, params persistence.save_everything(args, \ metrics_hist_all, \ model, \ model_dir, \ params, \ args.criterion, \ evaluate, \ best=best, \ ) if test_only: #we're done break if args.criterion in metrics_hist.keys(): if early_stop(metrics_hist, args.criterion, args.patience): #stop training, do tests on test and train sets, and then stop the script print("%s hasn't improved in %d epochs, early stopping..." % (args.criterion, args.patience)) test_only = True args.test_model = '%s/model_best_%s.pth' % (model_dir, args.criterion) model = tools.pick_model(args, dicts) return epoch + 1