def train_epochs(args, model, optimizer, params, dicts):
"""
Main loop. does train and test
"""
metrics_hist = defaultdict(lambda: [])
metrics_hist_te = defaultdict(lambda: [])
metrics_hist_tr = defaultdict(lambda: [])
test_only = args.test_model is not None
#train for n_epochs unless criterion metric does not improve for [patience] epochs
for epoch in range(args.n_epochs):
#only test on train/test set on very last epoch
if epoch == 0 and not args.test_model:
model_dir = os.path.join(
MODEL_DIR, '_'.join(
[args.model,
time.strftime('%b_%d_%H:%M', time.gmtime())]))
os.mkdir(model_dir)
elif args.test_model:
model_dir = os.getcwd(
) #just save things to where this script was called
metrics_all = one_epoch(model, optimizer, epoch, args.n_epochs,
args.batch_size, args.data_path, test_only,
dicts, model_dir, args.samples, args.gpu,
args.debug, args.quiet)
# DISTRIBUTING results from metrics_all to respective dicts
for name in metrics_all[0].keys():
metrics_hist[name].append(metrics_all[0][name])
for name in metrics_all[1].keys():
metrics_hist_te[name].append(metrics_all[1][name])
for name in metrics_all[2].keys():
metrics_hist_tr[name].append(metrics_all[2][name])
metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr)
#save metrics, model, params
persistence.save_everything(
args, metrics_hist_all, model, model_dir, params, args.criterion
) # SHOULD SAVE MODEL PARAMS AT EACH EPOCH, BELIEVE IS HAPPENING
if test_only:
break
if args.criterion in metrics_hist.keys():
if early_stop(metrics_hist, args.criterion, args.patience):
#stop training, do tests on test and train sets, and then stop the script
print("%s hasn't improved in %d epochs, early stopping..." %
(args.criterion, args.patience))
test_only = True
model = torch.load(
'%s/model_best_%s.pth' %
(model_dir,
args.criterion)) # LOADING BEST MODEL FOR FINAL TEST
return epoch + 1
def train_epochs(args, model, optimizer, params, dicts):
"""
Main loop. does train and test
"""
metrics_hist = defaultdict(lambda: [])
metrics_hist_te = defaultdict(lambda: [])
metrics_hist_tr = defaultdict(lambda: [])
test_only = args.test_model is not None
evaluate = args.test_model is not None
#train for n_epochs unless criterion metric does not improve for [patience] epochs
for epoch in range(args.n_epochs):
#only test on train/test set on very last epoch
if epoch == 0 and not args.test_model:
model_dir = os.path.join(
MODEL_DIR, '_'.join([
args.model,
time.strftime('%b_%d_%H:%M:%S', time.localtime())
]))
os.mkdir(model_dir)
elif args.test_model:
model_dir = os.path.dirname(os.path.abspath(args.test_model))
metrics_all = one_epoch(model, optimizer, args.Y, epoch, args.n_epochs,
args.batch_size, args.data_path, args.version,
test_only, dicts, model_dir, args.samples,
args.gpu, args.quiet)
for name in metrics_all[0].keys():
metrics_hist[name].append(metrics_all[0][name])
for name in metrics_all[1].keys():
metrics_hist_te[name].append(metrics_all[1][name])
for name in metrics_all[2].keys():
metrics_hist_tr[name].append(metrics_all[2][name])
metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr)
#save metrics, model, params
persistence.save_everything(args, metrics_hist_all, model, model_dir,
params, args.criterion, evaluate)
if test_only:
#we're done
break
if args.criterion in metrics_hist.keys():
if early_stop(metrics_hist, args.criterion, args.patience):
#stop training, do tests on test and train sets, and then stop the script
print("%s hasn't improved in %d epochs, early stopping..." %
(args.criterion, args.patience))
test_only = True
args.test_model = '%s/model_best_%s.pth' % (model_dir,
args.criterion)
model = tools.pick_model(args, dicts)
return epoch + 1
def train_epochs(args, model, optimizer, params, freq_params, dicts):
"""
Main loop. does train and test
"""
metrics_hist = defaultdict(lambda: [])
metrics_hist_te = defaultdict(lambda: [])
metrics_hist_tr = defaultdict(lambda: [])
test_only = args.test_model is not None
unseen_code_inds = set()
#train for n_epochs unless criterion metric does not improve for [patience] epochs
for epoch in range(args.n_epochs):
#only test on train/test set on very last epoch
if epoch == 0 and not args.test_model:
model_dir = os.path.join(
MODEL_DIR, '_'.join(
[args.model,
time.strftime('%b_%d_%H:%M', time.gmtime())]))
os.mkdir(model_dir)
elif args.test_model:
#just save things to where this script was called
model_dir = os.getcwd()
metrics_all, unseen_code_inds = one_epoch(
model, optimizer, args.Y, epoch, args.n_epochs, args.batch_size,
args.data_path, args.version, freq_params, test_only, dicts,
model_dir, unseen_code_inds, args.samples, args.gpu, args.debug,
args.quiet)
for name in metrics_all[0].keys():
metrics_hist[name].append(metrics_all[0][name])
for name in metrics_all[1].keys():
metrics_hist_te[name].append(metrics_all[1][name])
for name in metrics_all[2].keys():
metrics_hist_tr[name].append(metrics_all[2][name])
metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr)
#save metrics, model, params
persistence.save_everything(args, metrics_hist_all, model, model_dir,
params, args.criterion)
if test_only:
break
if args.criterion in metrics_hist.keys():
if early_stop(metrics_hist, args.criterion, args.patience):
#stop training, do tests on test and train sets, and then stop the script
print("%s hasn't improved in %d epochs, early stopping..." %
(args.criterion, args.patience))
test_only = True
model = torch.load('%s/model_best_%s.pth' %
(model_dir, args.criterion))
return epoch + 1
def train_epochs(args, model, optimizer, params, dicts, start_epoch):
"""
Main loop. does train and test
"""
metrics_hist = defaultdict(lambda: [])
metrics_hist_te = defaultdict(lambda: [])
metrics_hist_tr = defaultdict(lambda: [])
if args.reload_model: #get existing metrics up to model checkpoint:
with open(os.path.join(args.reload_model, 'metrics.json'), 'r') as f:
metrics_hist_all = json.load(f)
#this will be a little expensive-- iterate through and re-assign:
for el in metrics_hist_all:
if '_te' in el:
metrics_hist_te[re.sub(
'_te$', '', el)] = metrics_hist_all[el][:start_epoch]
elif '_tr' in el:
metrics_hist_tr[re.sub(
'_tr$', '', el)] = metrics_hist_all[el][:start_epoch]
else: #dev
metrics_hist[el] = metrics_hist_all[el][:start_epoch]
META_TEST = args.test_model is not None
test_only = args.test_model is not None
evaluate = args.test_model is not None
#train for n_epochs unless criterion metric does not improve for [patience] epochs
for epoch in range(
start_epoch, args.n_epochs
): #only train for _x_ more epochs from best-saved model*
#only test on train/test set on very last epoch
if epoch == start_epoch and not args.test_model:
model_dir = os.path.join(
MODEL_DIR, '_'.join([
args.model,
time.strftime('%b_%d_%H:%M', time.localtime())
]))
os.mkdir(model_dir)
#save model versioning (git) info:
repo = git.Repo(search_parent_directories=True)
branch = repo.active_branch.name
print("branch:", branch)
sha = repo.head.object.hexsha
print("SHA hash:", sha)
persistence.save_git_versioning_info(
model_dir, (branch, sha, args.description))
elif args.test_model:
model_dir = os.path.dirname(os.path.abspath(args.test_model))
metrics_all = one_epoch(args, model, optimizer, epoch, test_only,
dicts, model_dir, META_TEST)
for name in metrics_all[0].keys():
metrics_hist[name].append(metrics_all[0][name])
for name in metrics_all[1].keys():
metrics_hist_te[name].append(metrics_all[1][name])
for name in metrics_all[2].keys():
metrics_hist_tr[name].append(metrics_all[2][name])
metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr)
#save metrics, model, params
assert '-' not in args.criterion
persistence.save_everything(args, metrics_hist_all, model, model_dir,
params, optimizer, evaluate)
if test_only:
#we're done
break
if args.criterion in metrics_hist.keys():
if early_stop(metrics_hist, args.criterion, args.patience):
#stop training, do tests on test and train sets, and then stop the script
print("%s hasn't improved in %d epochs, early stopping..." %
(args.criterion, args.patience))
test_only = True
test_m = [
o for o in os.listdir(model_dir) if 'model_best' in o
]
assert (len(test_m) == 1)
args.test_model = os.path.join(model_dir, test_m[0])
model, _, _ = tools.pick_model(args, dicts, META_TEST)
return epoch - start_epoch + 1
def train_epochs(args, model, optimizer, params, dicts):
"""
Main loop. does train and test
"""
metrics_hist_train = defaultdict(list)
metrics_hist_dev = defaultdict(list)
metrics_hist_test = defaultdict(list)
if args.resume:
metrics_file = os.path.join(
os.path.dirname(os.path.abspath(args.resume)), 'metrics.json')
if os.path.exists(metrics_file):
with open(metrics_file, 'r') as metrics_fp:
metrics = json.load(metrics_fp)
metrics_hist_train.update({
k.replace('_train', ''): v
for k, v in metrics.items() if k.endswith('_train')
})
metrics_hist_dev.update({
k.replace('_dev', ''): v
for k, v in metrics.items() if k.endswith('_dev')
})
metrics_hist_test.update({
k.replace('_test', ''): v
for k, v in metrics.items() if k.endswith('_test')
})
test_only = args.test_model is not None
num_labels_fine = len(dicts['ind2c'])
num_labels_coarse = len(dicts['ind2c_coarse'])
epoch = 0 if args.resume is None else args.epoch
if not test_only:
dataset_train = MimicDataset(args.data_path, dicts, num_labels_fine,
num_labels_coarse, args.max_len)
dataset_dev = MimicDataset(args.data_path.replace('train', 'dev'),
dicts, num_labels_fine, num_labels_coarse,
args.max_len)
if args.resume is None:
model_dir = os.path.join(
args.models_dir,
'_'.join([args.model,
time.strftime('%Y-%m-%d_%H:%M:%S')]))
os.mkdir(model_dir)
else:
model_dir = os.path.dirname(os.path.abspath(args.resume))
else:
model_dir = os.path.dirname(os.path.abspath(args.test_model))
#tensorboard = Tensorboard(model_dir)
#train for n_epochs unless criterion metric does not improve for [patience] epochs
for epoch in range(epoch, args.n_epochs if not test_only else 0):
losses = train(model, optimizer, args.Y, epoch, args.batch_size,
args.embed_desc, dataset_train, args.shuffle, args.gpu,
dicts)
loss = np.mean(losses)
metrics_train = {'loss': loss}
fold = 'dev'
#evaluate on dev
with torch.no_grad():
metrics_dev, _, _, _ = test(model, args.Y, epoch, dataset_dev,
args.batch_size, args.embed_desc, fold,
args.gpu, dicts, model_dir)
for name, val in metrics_train.items():
#tensorboard.log_scalar('%s_train' % (name), val, epoch)
metrics_hist_train[name].append(metrics_train[name])
metrics_hist_train.update({'epochs': epoch + 1})
for name, val in metrics_dev.items():
#tensorboard.log_scalar('%s_dev' % (name), val, epoch)
metrics_hist_dev[name].append(metrics_dev[name])
metrics_hist_all = (metrics_hist_train, metrics_hist_dev, None)
#save metrics, model, optimizer state, params
persistence.save_everything(args,
dicts,
metrics_hist_all,
model,
optimizer,
model_dir,
params,
args.criterion,
evaluate=False,
test_only=False)
if args.criterion is not None:
if early_stop(metrics_hist_dev, args.criterion, args.patience):
#stop training, evaluate on test set and then stop the script
print('{} hasn\'t improved in {} epochs, early stopping...'.
format(args.criterion, args.patience))
break
fold = 'test'
print("\nevaluating on test")
dataset_train = None
dataset_dev = None
del dataset_train, dataset_dev
if not test_only:
model_best_sd = torch.load(
os.path.join(model_dir,
'model_best_{}.pth'.format(args.criterion)))
model.load_state_dict(model_best_sd)
if args.gpu:
model.cuda()
dataset_test = MimicDataset(args.data_path.replace('train', 'test'), dicts,
num_labels_fine, num_labels_coarse,
args.max_len)
with torch.no_grad():
metrics_test, metrics_codes, metrics_inst, hadm_ids = test(
model, args.Y, epoch, dataset_test, args.batch_size,
args.embed_desc, fold, args.gpu, dicts, model_dir)
for name, val in metrics_test.items():
#if not test_only:
# tensorboard.log_scalar('%s_test' % (name), val, epoch)
metrics_hist_test[name].append(metrics_test[name])
metrics_hist_all = (metrics_hist_train, metrics_hist_dev,
metrics_hist_test)
#tensorboard.close()
#save metrics, model, params
persistence.save_everything(args,
dicts,
metrics_hist_all,
model,
optimizer,
model_dir,
params,
args.criterion,
metrics_codes=metrics_codes,
metrics_inst=metrics_inst,
hadm_ids=hadm_ids,
evaluate=True,
test_only=test_only)
return epoch + 1, metrics_hist_test
def train_epochs(args, model, optimizer, params, dicts, scheduler,
labels_weight):
"""
Main loop. does train and test
"""
dev_acc_macro, dev_prec_macro, dev_rec_macro, dev_f1_macro = 0.0, 0.0, 0.0, 0.0
dev_acc_micro, dev_prec_micro, dev_rec_micro, dev_f1_micro = 0.0, 0.0, 0.0, 0.0
dev_rec_at_5, dev_prec_at_5, dev_f1_at_5 = 0.0, 0.0, 0.0
dev_auc_macro, dev_auc_micro = 0.0, 0.0
dev_epoch, dev_loss = 0.0, 0.0
test_acc_macro, test_prec_macro, test_rec_macro, test_f1_macro = 0.0, 0.0, 0.0, 0.0
test_acc_micro, test_prec_micro, test_rec_micro, test_f1_micro = 0.0, 0.0, 0.0, 0.0
test_rec_at_5, test_prec_at_5, test_f1_at_5 = 0.0, 0.0, 0.0
test_auc_macro, test_auc_micro = 0.0, 0.0
test_epoch, test_loss = 0.0, 0.0
metrics_hist = defaultdict(lambda: [])
metrics_hist_te = defaultdict(lambda: [])
metrics_hist_tr = defaultdict(lambda: [])
test_only = args.test_model is not None
evaluate = args.test_model is not None
# train for n_epochs unless criterion metric does not improve for [patience] epochs
for epoch in range(args.n_epochs):
best = False
#only test on train/test set on very last epoch
if epoch == 0 and not args.test_model:
model_dir = os.path.join(
MODEL_DIR, '_'.join([
args.model,
time.strftime('%b_%d_%H:%M:%S', time.localtime())
]))
os.makedirs(model_dir)
elif args.test_model:
model_dir = os.path.dirname(os.path.abspath(args.test_model))
metrics_all = one_epoch(args, \
model, \
optimizer, \
args.Y, \
epoch, \
args.n_epochs, \
args.batch_size, \
args.data_path, \
args.version, \
test_only, \
dicts, \
model_dir, \
args.samples, \
args.gpu, \
args.quiet, \
scheduler, \
labels_weight, \
)
for name in metrics_all[0].keys():
metrics_hist[name].append(metrics_all[0][name])
for name in metrics_all[1].keys():
metrics_hist_te[name].append(metrics_all[1][name])
for name in metrics_all[2].keys():
metrics_hist_tr[name].append(metrics_all[2][name])
metrics_hist_all = (metrics_hist, metrics_hist_te, metrics_hist_tr)
if metrics_hist_te['auc_micro'][-1] >= test_auc_micro:
test_acc_macro = metrics_hist_te['acc_macro'][-1]
test_prec_macro = metrics_hist_te['prec_macro'][-1]
test_rec_macro = metrics_hist_te['rec_macro'][-1]
test_f1_macro = metrics_hist_te['f1_macro'][-1]
test_acc_micro = metrics_hist_te['acc_micro'][-1]
test_prec_micro = metrics_hist_te['prec_micro'][-1]
test_rec_micro = metrics_hist_te['rec_micro'][-1]
test_f1_micro = metrics_hist_te['f1_micro'][-1]
if not args.Y == 'full':
test_rec_at_5 = metrics_hist_te['rec_at_5'][-1]
test_prec_at_5 = metrics_hist_te['prec_at_5'][-1]
test_f1_at_5 = metrics_hist_te['f1_at_5'][-1]
test_auc_macro = metrics_hist_te['auc_macro'][-1]
test_auc_micro = metrics_hist_te['auc_micro'][-1]
test_loss = metrics_hist_te['loss_test'][-1]
test_epoch = epoch
if metrics_hist['auc_micro'][-1] >= dev_auc_micro:
dev_acc_macro = metrics_hist['acc_macro'][-1]
dev_prec_macro = metrics_hist['prec_macro'][-1]
dev_rec_macro = metrics_hist['rec_macro'][-1]
dev_f1_macro = metrics_hist['f1_macro'][-1]
dev_acc_micro = metrics_hist['acc_micro'][-1]
dev_prec_micro = metrics_hist['prec_micro'][-1]
dev_rec_micro = metrics_hist['rec_micro'][-1]
dev_f1_micro = metrics_hist['f1_micro'][-1]
if not args.Y == 'full':
dev_rec_at_5 = metrics_hist['rec_at_5'][-1]
dev_prec_at_5 = metrics_hist['prec_at_5'][-1]
dev_f1_at_5 = metrics_hist['f1_at_5'][-1]
dev_auc_macro = metrics_hist['auc_macro'][-1]
dev_auc_micro = metrics_hist['auc_micro'][-1]
dev_loss = metrics_hist['loss_dev'][-1]
dev_epoch = epoch
best = True
print()
print('-' * 19 + ' Best (Dev) ' + '-' * 19)
print('Best Epoch: %d' % dev_epoch)
print()
print("[MACRO] accuracy, precision, recall, f-measure, AUC")
print("%.4f, %.4f, %.4f, %.4f, %.4f" %
(dev_acc_macro, dev_prec_macro, dev_rec_macro, dev_f1_macro,
dev_auc_macro))
print("[MICRO] accuracy, precision, recall, f-measure, AUC")
print("%.4f, %.4f, %.4f, %.4f, %.4f" %
(dev_acc_micro, dev_prec_micro, dev_rec_micro, dev_f1_micro,
dev_auc_micro))
print('rec_at_5: %.4f' % dev_rec_at_5)
print('prec_at_5: %.4f' % dev_prec_at_5)
print()
print('Dev loss: %.4f' % dev_loss)
print()
print('-' * 51)
print()
print('-' * 19 + ' Best (Test) ' + '-' * 19)
print('Best Epoch: %d' % test_epoch)
print()
print("[MACRO] accuracy, precision, recall, f-measure, AUC")
print("%.4f, %.4f, %.4f, %.4f, %.4f" %
(test_acc_macro, test_prec_macro, test_rec_macro, test_f1_macro,
test_auc_macro))
print("[MICRO] accuracy, precision, recall, f-measure, AUC")
print("%.4f, %.4f, %.4f, %.4f, %.4f" %
(test_acc_micro, test_prec_micro, test_rec_micro, test_f1_micro,
test_auc_micro))
print('rec_at_5: %.4f' % test_rec_at_5)
print('prec_at_5: %.4f' % test_prec_at_5)
print()
print('Test loss: %.4f' % test_loss)
print()
print('-' * 51)
# save metrics, model, params
persistence.save_everything(args, \
metrics_hist_all, \
model, \
model_dir, \
params, \
args.criterion, \
evaluate, \
best=best, \
)
if test_only:
#we're done
break
if args.criterion in metrics_hist.keys():
if early_stop(metrics_hist, args.criterion, args.patience):
#stop training, do tests on test and train sets, and then stop the script
print("%s hasn't improved in %d epochs, early stopping..." %
(args.criterion, args.patience))
test_only = True
args.test_model = '%s/model_best_%s.pth' % (model_dir,
args.criterion)
model = tools.pick_model(args, dicts)
return epoch + 1
