dev_batch = DataLoader(opt['data_dir'] + '/dev.json',
opt['batch_size'],
opt,
vocab,
evaluation=True)
model_id = opt['id'] if len(opt['id']) > 1 else '0' + opt['id']
model_save_dir = opt['save_dir'] + '/' + model_id
opt['model_save_dir'] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
helper.save_config(opt, model_save_dir + '/config.json', verbose=True)
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
model_save_dir + '/' + opt['log'],
header="# epoch\ttrain_loss\tdev_loss\tdev_score\tbest_dev_score")
# print model info
helper.print_config(opt)
# model
if not opt['load']:
trainer = GCNTrainer(opt, emb_matrix=emb_matrix)
else:
# load pretrained model
model_file = opt['model_file']
print("Loading model from {}".format(model_file))
model_opt = torch_utils.load_config(model_file)
model_opt['optim'] = opt['optim']
trainer = GCNTrainer(model_opt)
opt['batch_size'],
opt,
vocab,
evaluation=True,
pattern_file=opt['pattern_file'])
model_id = opt['id'] if len(opt['id']) > 1 else '0' + opt['id']
model_save_dir = opt['save_dir'] + '/' + model_id
opt['model_save_dir'] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
helper.save_config(opt, model_save_dir + '/config.json', verbose=True)
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
model_save_dir + '/' + opt['log'],
header="# epoch\ttrain_loss\tdev_loss\tdev_f1\ttest_loss\ttest_f1")
# print model info
helper.print_config(opt)
# model
base_model = BaseModel(opt=opt, emb_matrix=emb_matrix)
id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()])
dev_f1_history = []
current_lr = opt['lr']
global_step = 0
global_start_time = time.time()
format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}'
def transre_search(ffn, connect, hidden_dim, trans_layers, multi_heads,
ffn_ex_size, initial, final):
opt['weighted'] = False
opt['rnn'] = False
opt['ffn'] = ffn
opt['connect'] = connect
opt['hidden_dim'] = hidden_dim
opt['trans_layers'] = trans_layers
opt['multi_heads'] = multi_heads
opt['ffn_ex_size'] = ffn_ex_size
opt['initial'] = initial
opt['final'] = final
id = opt['id'] if len(opt['id']) > 1 else '0' + opt['id']
model_name =str (opt['optim']) + '_' + str (opt['lr']) + str (ffn) + '_' +str(connect)+"_"\
+ str (hidden_dim) + '_' + str (trans_layers) + '_' + str (multi_heads) + '_' + \
str (ffn_ex_size)+'_'+str(initial)+'_'+str(final)
model_name = model_name + '' + str(opt['memo'])
model_name = str(id) + "_" + model_name
model_save_dir = opt['save_dir'] + '/' + model_name
opt['model_save_dir'] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
helper.save_config(opt, model_save_dir + '/config.json', verbose=True)
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
model_save_dir + '/' + opt['log'],
header="# epoch\ttrain_loss\tdev_loss\tdev_score\tbest_dev_score")
helper.print_config(opt)
if not opt['load']:
trainer = TransTrainer(opt, emb_matrix=emb_matrix)
else:
# load pre-train model
model_file = opt['model_file']
print("Loading model from {}".format(model_file))
model_opt = torch_utils.load_config(model_file)
model_opt['optim'] = opt['optim']
trainer = TransTrainer(model_opt)
trainer.load(model_file)
id2label = dict([(v, k) for k, v in label2id.items()
]) # the classification result
dev_score_history = []
dev_loss_history = []
current_lr = opt['lr']
global_step = 0
format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}'
max_steps = len(train_batch) * opt['num_epoch']
best_result = "unknown"
file_logger.log(str(opt['memo']))
for epoch in range(1, opt['num_epoch'] + 1):
train_loss = 0
epoch_start_time = time.time()
for i, batch in enumerate(train_batch):
start_time = time.time()
global_step += 1
loss, norm = trainer.update(batch)
train_loss += loss
if global_step % opt['log_step'] == 0:
duration = time.time() - start_time
print(
format_str.format(datetime.now(), global_step, max_steps,
epoch, opt['num_epoch'], loss, duration,
current_lr))
print("Evaluating on dev set...")
predictions = []
dev_loss = 0
for i, batch in enumerate(dev_batch):
preds, _, loss, _ = trainer.predict(batch)
predictions += preds
dev_loss += loss
predictions = [id2label[p] for p in predictions]
train_loss = train_loss / train_batch.num_examples * opt[
'batch_size'] # avg loss per batch
dev_loss = dev_loss / dev_batch.num_examples * opt['batch_size']
acc, dev_p, dev_r, dev_f1 = scorer.score(dev_batch.gold(), predictions)
print(
"epoch {}: train_loss = {:.6f}, dev_loss = {:.6f}, dev_f1 = {:.4f}"
.format(epoch, train_loss, dev_loss, dev_f1))
dev_score = dev_f1
file_logger.log("{}\t{:.3f}\t{:.6f}\t{:.6f}\t{:.4f}\t{:.4f}".format(
epoch, acc, train_loss, dev_loss, dev_score,
max([dev_score] + dev_score_history)))
# save
model_file = model_save_dir + '/checkpoint_epoch_{}.pt'.format(epoch)
trainer.save(model_file, epoch)
if epoch == 1 or dev_score > max(dev_score_history):
copyfile(model_file, model_save_dir + '/best_model.pt')
best_result = (model_name, dev_score)
print("new best model saved.")
file_logger.log(
"new best model saved at epoch {}: {:.2f}\t{:.2f}\t{:.2f}".
format(epoch, dev_p * 100, dev_r * 100, dev_score * 100))
if epoch % opt['save_epoch'] != 0:
os.remove(model_file)
# lr schedule
if len(dev_score_history
) > opt['decay_epoch'] and dev_score <= dev_score_history[
-1] and opt['optim'] in ['sgd', 'adagrad', 'adadelta']:
current_lr *= opt['lr_decay']
trainer.update_lr(current_lr)
dev_score_history += [dev_score]
dev_loss_history += [dev_loss]
epoch_end_time = time.time()
print("epoch time {:.3f}".format(epoch_end_time - epoch_start_time))
return best_result
dicts['token'] = token_vocab['w2i']
# load training set and test set
print("Loading data from {} with batch size {}...".format(
args.dataset, args.batch_size))
train_batch = DataLoader('./dataset/' + args.dataset + '/train.json',
args.batch_size, args, dicts)
test_batch = DataLoader('./dataset/' + args.dataset + '/test.json',
args.batch_size, args, dicts)
# create the folder for saving the best models and log file
model_save_dir = args.save_dir
helper.ensure_dir(model_save_dir, verbose=True)
file_logger = helper.FileLogger(
model_save_dir + '/' + args.log,
header="# epoch\ttrain_loss\ttest_loss\ttrain_acc\ttest_acc\ttest_f1")
# create the model
trainer = GCNTrainer(args, emb_matrix=emb_matrix)
# start training
train_acc_history, train_loss_history, test_loss_history, f1_score_history = [], [], [], [0.]
test_acc_history = [0.]
for epoch in range(1, args.num_epoch + 1):
train_loss, train_acc, train_step = 0., 0., 0
for i, batch in enumerate(train_batch):
loss, acc = trainer.update(batch)
train_loss += loss
train_acc += acc
train_step += 1
ent_vocab,
first_ent_span_token=args.use_first_ent_span_tok,
evaluation=True,
)
model_id = opt["id"] if len(opt["id"]) > 1 else "0" + opt["id"]
model_save_dir = opt["save_dir"] + "/" + model_id
opt["model_save_dir"] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
helper.save_config(opt, model_save_dir + "/config.json", verbose=True)
vocab.save(model_save_dir + "/vocab.pkl")
if args.use_ctx_ent:
ent_vocab.save(model_save_dir + "/ent_vocab.pkl")
file_logger = helper.FileLogger(model_save_dir + "/" + opt["log"],
header="# epoch\ttrain_loss\tdev_loss\tdev_f1")
# print model info
helper.print_config(opt)
# model
model = RelationModel(opt,
emb_matrix=emb_matrix,
ent_emb_matrix=ent_emb_matrix)
id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()])
dev_f1_history = []
current_lr = opt["lr"]
global_step = 0
global_start_time = time.time()
assert emb_matrix.shape[1] == opt['emb_dim']
# load data
print("Loading data from {} with batch size {}...".format(opt['data_dir'], opt['batch_size']))
train_batch = DataLoader(opt['data_dir'] + "/"+opt['train_file'], opt['batch_size'], opt, vocab, opt['life'], evaluation=False)
dev_batch = DataLoader(opt['data_dir'] + "/"+opt['dev_file'], opt['batch_size'], opt, vocab, opt['life'], evaluation=True)
model_id = opt['id'] if len(opt['id']) > 1 else '0' + opt['id']
model_save_dir = opt['save_dir'] + '/' + model_id
opt['model_save_dir'] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
helper.save_config(opt, model_save_dir + '/config.json', verbose=True)
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(model_save_dir + '/' + opt['log'], header="# epoch\ttrain_loss\tdev_loss\tdev_f1\tPrecision\tRecall")
# print model info
helper.print_config(opt)
# model
model = RelationModel(opt, emb_matrix=None if opt['bert'] else emb_matrix)
if opt['life']:
id2label = dict([(v,k) for k,v in constant.LIFE_LABEL_TO_ID.items()])
if not opt['life']:
id2label = dict([(v,k) for k,v in constant.LABEL_TO_ID.items()])
dev_f1_history = []
current_lr = opt['lr']
global_step = 0
def train_unbiased_model(args, biased_batch_probs):
# make opt
opt = vars(args)
opt["num_class"] = len(constant.LABEL_TO_ID)
# load vocab
vocab_file = opt['vocab_dir'] + '/vocab.pkl'
vocab = Vocab(vocab_file, load=True)
opt['vocab_size'] = vocab.size
emb_file = opt['vocab_dir'] + '/embedding.npy'
emb_matrix = np.load(emb_file)
assert emb_matrix.shape[0] == vocab.size
assert emb_matrix.shape[1] == opt['emb_dim']
# load data
print("Loading data from {} with batch size {}...".format(
opt["data_dir"], opt["batch_size"]))
train_batch = DataLoader(
opt["data_dir"] + "/" + args.data_name,
opt["batch_size"],
opt,
vocab,
evaluation=False,
)
dev_batch = DataLoader(opt["data_dir"] + "/dev.json",
opt["batch_size"],
opt,
vocab,
evaluation=True)
model_id = opt["id"] if len(opt["id"]) > 1 else "0" + opt["id"]
model_save_dir = opt["save_dir"] + "/" + model_id
opt["model_save_dir"] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
helper.save_config(opt, model_save_dir + "/config.json", verbose=True)
vocab.save(model_save_dir + "/vocab.pkl")
file_logger = helper.FileLogger(
model_save_dir + "/" + opt["log"],
header="# epoch\ttrain_loss\tdev_loss\tdev_f1")
# print model info
helper.print_config(opt)
# model
model = RelationModel(opt, emb_matrix=emb_matrix)
id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()])
dev_f1_history = []
current_lr = opt["lr"]
global_step = 0
global_start_time = time.time()
format_str = (
"{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}"
)
max_steps = len(train_batch) * opt["num_epoch"]
# start training
for epoch in range(1, opt["num_epoch"] + 1):
train_loss = 0
for i, batch in enumerate(train_batch):
start_time = time.time()
global_step += 1
loss = model.update(batch,
torch.tensor(biased_batch_probs[i]).cuda())
train_loss += loss
if global_step % opt["log_step"] == 0:
duration = time.time() - start_time
print(
format_str.format(
datetime.now(),
global_step,
max_steps,
epoch,
opt["num_epoch"],
loss,
duration,
current_lr,
))
# eval on dev
print("Evaluating on dev set...")
predictions = []
dev_loss = 0
for i, batch in enumerate(dev_batch):
preds, _, loss = model.predict(batch)
predictions += preds
dev_loss += loss
predictions = [id2label[p] for p in predictions]
dev_p, dev_r, dev_f1 = scorer.score(dev_batch.gold(), predictions)
f = open("label.txt", "w+")
f.write(str(dev_batch.gold()))
f.close()
train_loss = (train_loss / train_batch.num_examples * opt["batch_size"]
) # avg loss per batch
dev_loss = dev_loss / dev_batch.num_examples * opt["batch_size"]
print(
"epoch {}: train_loss = {:.6f}, dev_loss = {:.6f}, dev_f1 = {:.4f}"
.format(epoch, train_loss, dev_loss, dev_f1))
file_logger.log("{}\t{:.6f}\t{:.6f}\t{:.4f}".format(
epoch, train_loss, dev_loss, dev_f1))
# save
model_file = model_save_dir + "/checkpoint_epoch_{}.pt".format(epoch)
model.save(model_file, epoch)
if epoch == 1 or dev_f1 > max(dev_f1_history):
copyfile(model_file, model_save_dir + "/best_model.pt")
print("new best model saved.")
if epoch % opt["save_epoch"] != 0:
os.remove(model_file)
# lr schedule
if (len(dev_f1_history) > 10 and dev_f1 <= dev_f1_history[-1]
and opt["optim"] in ["sgd", "adagrad"]):
current_lr *= opt["lr_decay"]
model.update_lr(current_lr)
dev_f1_history += [dev_f1]
print("")
print("Training ended with {} epochs.".format(epoch))
opt = vars(args)
label2id = constant.LABEL_TO_ID
opt['num_class'] = len(label2id)
# print opt
helper.print_config(opt)
label2id = constant.LABEL_TO_ID
id2label = dict([(v, k) for k, v in label2id.items()])
# model save dir
helper.ensure_dir(opt['save_dir'], verbose=True)
# save config
helper.save_config(opt, opt['save_dir'] + '/config.json', verbose=True)
file_logger = helper.FileLogger(
opt['save_dir'] + '/' + opt['log'],
header="# epoch\ttrain_loss\tdev_loss\ttrain_ACC\ttest_ACC\tF1")
# load data
print("Loading data from {} with batch size {} ...".format(
opt['data_dir'], opt['batch_size']))
train_batch = DataLoader(opt['data_dir'] + '/train.tsv', opt['batch_size'],
opt)
test_batch = DataLoader(opt['data_dir'] + '/test.tsv', opt['batch_size'], opt)
# build model
print('Building model...')
trainer = MyTrainer(opt)
train_acc_history, train_loss_history, test_loss_history, f1_score_history = [], [], [], [0.]
test_acc_history = [0.]
args.use_cuda = False
elif args.use_cuda:
torch.cuda.manual_seed(args.seed)
opt = vars(args)
# print model info
helper.print_config(opt)
helper.ensure_dir(opt["model_save_dir"], verbose=True)
# save model config
helper.save_config(opt,
opt["model_save_dir"] + "/" + opt["id"] + '.config',
verbose=True)
# record training log
file_logger = helper.FileLogger(
opt["model_save_dir"] + '/' + opt['id'] + ".log",
header="# epoch\ttrain_loss\tprecision5\tNDCG5\tMAP5\tprecision7"
"\tNDCG7\tMAP7\tprecision10\tNDCG10\tMAP10")
preprocess = Preprocess(opt)
print("Preprocess is done.")
print("Create model TaNP...")
opt['uf_dim'] = preprocess.uf_dim
opt['if_dim'] = preprocess.if_dim
trainer = Trainer(opt)
if opt['use_cuda']:
trainer.cuda()
model_filename = "{}/{}.pt".format(opt['model_save_dir'], opt["id"])
def main():
# set top-level random seeds
torch.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
if args.cpu:
args.cuda = False
elif args.cuda:
# force random seed for reproducibility
# also apply same seed to numpy in every file
torch.backends.cudnn.deterministic = True
torch.cuda.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# make opt
opt = vars(args)
opt['num_class'] = len(constant.LABEL_TO_ID)
# load vocab
vocab_file = opt['vocab_dir'] + '/vocab.pkl'
vocab = Vocab(vocab_file, load=True)
# in some previous experiments we saw that lower vocab size can improve performance
# but it was in a completely different project although on the same data
# here it seems it's much harder to get this to work
# uncomment the following line if this is solved:
# new_vocab_size = 30000
opt['vocab_size'] = vocab.size
emb_file = opt['vocab_dir'] + '/embedding.npy'
emb_matrix = np.load(emb_file)
assert emb_matrix.shape[0] == vocab.size
assert emb_matrix.shape[1] == opt['emb_dim']
# load data
print("Loading data from {} with batch size {}...".format(
opt['data_dir'], opt['batch_size']))
train_batch = DataLoader(opt['data_dir'] + '/train.json',
opt['batch_size'],
opt,
vocab,
evaluation=False)
dev_batch = DataLoader(opt['data_dir'] + '/dev.json',
opt['batch_size'],
opt,
vocab,
evaluation=True)
model_id = opt['id'] if len(opt['id']) > 1 else '0' + opt['id']
model_save_dir = opt['save_dir'] + '/' + model_id
opt['model_save_dir'] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
helper.save_config(opt, model_save_dir + '/config.json', verbose=True)
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
model_save_dir + '/' + opt['log'],
header="# epoch\ttrain_loss\tdev_loss\tdev_p\tdev_r\tdev_f1")
# print model info
helper.print_config(opt)
# model
model = RelationModel(opt, emb_matrix=emb_matrix)
id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()])
dev_f1_history = []
current_lr = opt['lr']
global_step = 0
format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}'
max_steps = len(train_batch) * opt['num_epoch']
# setup the scheduler for lr decay
# this doesn't seem to work well compared to what we already have
# scheduler = ReduceLROnPlateau(model.optimizer, mode='min', factor=opt['lr_decay'], patience=1)
# start training
for epoch in range(1, opt['num_epoch'] + 1):
# TODO: if lr warmup is used, the lr console output is not updated
print(
"Current params: " + " heads-" + str(opt["n_head"]) +
" enc_layers-" + str(opt["num_layers_encoder"]),
" drop-" + str(opt["dropout"]) + " scaled_drop-" +
str(opt["scaled_dropout"]) + " lr-" + str(opt["lr"]),
" lr_decay-" + str(opt["lr_decay"]) + " max_grad_norm-" +
str(opt["max_grad_norm"]))
print(
" weight_no_rel-" + str(opt["weight_no_rel"]) + " weight_rest-" +
str(opt["weight_rest"]) + " attn-" + str(opt["attn"]) +
" attn_dim-" + str(opt["attn_dim"]),
" obj_sub_pos-" + str(opt["obj_sub_pos"]) + " new_residual-" +
str(opt["new_residual"]))
print(
" use_batch_norm-" + str(opt["use_batch_norm"]) +
" relative_positions-" + str(opt["relative_positions"]),
" decay_epoch-" + str(opt["decay_epoch"]) + " use_lemmas-" +
str(opt["use_lemmas"]), " hidden_self-" + str(opt["hidden_self"]))
train_loss = 0
for i, batch in enumerate(train_batch):
start_time = time.time()
global_step += 1
loss = model.update(batch)
train_loss += float(loss)
if global_step % opt['log_step'] == 0:
duration = time.time() - start_time
print(
format_str.format(datetime.now(), global_step, max_steps,
epoch, opt['num_epoch'], loss, duration,
current_lr))
# do garbage collection,
# as per https://discuss.pytorch.org/t/best-practices-for-maximum-gpu-utilization/13863/6
del loss
# eval on dev
print("Evaluating on dev set...")
predictions = []
dev_loss = 0
for i, batch in enumerate(dev_batch):
preds, _, loss = model.predict(batch)
predictions += preds
dev_loss += float(loss)
del loss
predictions = [id2label[p] for p in predictions]
dev_p, dev_r, dev_f1 = scorer.score(dev_batch.gold(), predictions)
train_loss = train_loss / train_batch.num_examples * opt[
'batch_size'] # avg loss per batch
dev_loss = dev_loss / dev_batch.num_examples * opt['batch_size']
print(
"epoch {}: train_loss = {:.6f}, dev_loss = {:.6f}, dev_f1 = {:.4f}".format(epoch, \
train_loss, dev_loss, dev_f1)
)
file_logger.log("{}\t{:.6f}\t{:.6f}\t{:.4f}\t{:.4f}\t{:.4f}".format(
epoch, train_loss, dev_loss, dev_p, dev_r, dev_f1))
# save
model_file = model_save_dir + '/checkpoint_epoch_{}.pt'.format(epoch)
model.save(model_file, epoch)
if epoch == 1 or dev_f1 > max(dev_f1_history):
copyfile(model_file, model_save_dir + '/best_model.pt')
print("new best model saved.")
if epoch % opt['save_epoch'] != 0:
os.remove(model_file)
# reduce learning rate if it stagnates by a certain decay rate and within given epoch patience
# this for some reason works worth than the implementation we have afterwards
# scheduler.step(dev_loss)
if opt["optim"] != "noopt_adam" and opt["optim"] != "noopt_nadam":
# do warm_up_for sgd only instead of adam
do_warmup_trick = False
if do_warmup_trick:
# print("do_warmup_trick")
# 1 and 5 first worked kind of
# 10 and 15
current_lr = 10 * (360**(-0.5) *
min(epoch**(-0.5), epoch * 15**(-1.5)))
# print("current_lr", current_lr)
model.update_lr(current_lr)
else:
# decay schedule # 15 is best!
# simulate patience of x epochs
if len(dev_f1_history
) > opt['decay_epoch'] and dev_f1 <= dev_f1_history[-1]:
current_lr *= opt['lr_decay']
model.update_lr(current_lr)
# else, update the learning rate in torch_utils.py
dev_f1_history += [dev_f1]
print("")
print("Training ended with {} epochs.".format(epoch))
args.dicts = dicts
# load training set and test set
print("Loading data from {} with batch size {}...".format(
args.dataset, args.batch_size))
train_batch = DataLoader('./dataset/' + args.dataset + '/train.json',
args.batch_size, args, dicts)
test_batch = DataLoader('./dataset/' + args.dataset + '/test.json',
args.batch_size, args, dicts)
# create the folder for saving the best models and log file
model_save_dir = args.save_dir
helper.ensure_dir(model_save_dir, verbose=True)
file_logger = helper.FileLogger(
model_save_dir + '/' + args.log,
header=
"# epoch\ttrain_loss\ttrain_loss_er\ttrain_loss_rc\tdev_loss\tdev_loss_er\tdev_loss_rc\tP\tR\tF1"
)
print('Building model...')
trainer = MyTrainer(args, emb_matrix=emb_matrix)
# start training
e_stop = 0
train_loss_his, train_loss_ner_his, train_loss_rc_his, test_loss_his, test_loss_ner_his, test_loss_rc_his, P_his, R_his, F1_his = [], [], [], [], [], [], [], [], []
for epoch in range(1, args.num_epoch + 1):
e_stop += 1
if e_stop > args.early_stop:
break
train_loss, train_loss_ner, train_loss_rc, train_step = 0., 0., 0., 0
f_auc, dev_f, dev_p, dev_r, maxac, maxj = evaluate.f_compute(gold_answer,predictions_list)
_, intra_dev_f, intra_dev_p, intra_dev_r, _, _ = evaluate.f_compute(intra_answer, intra_predictions)
_, inter_dev_f, inter_dev_p, inter_dev_r, _, _ = evaluate.f_compute(inter_answer, inter_predictions)
f_test_pred = open('./output_test_prediction.txt', "w")
for i in range(len(predictions_list)):
if predictions_list[i] >= maxj:
f_test_pred.write('\t'.join([doc_id_list[i], 'CID', mesh_id_list[i][0], mesh_id_list[i][1], str(distance_list[i])]) + '\n')
f_test_pred.close()
print('Test results: Prec:{} | Rec:{} | F1:{} '.format(dev_p, dev_r, dev_f))
print('Test results for intra-sentence level: Prec:{} | Rec:{} | F1:{} '.format(intra_dev_p, intra_dev_r, intra_dev_f))
print('Test results for inter-sentence level: Prec:{} | Rec:{} | F1:{} '.format(inter_dev_p, inter_dev_r, inter_dev_f))
test_result_file = model_save_dir + '/' + 'test_result.txt'
test_file_logger = helper.FileLogger(test_result_file)
test_file_logger.log('Test results: Prec:{} | Rec:{} | F1:{} '.format(dev_p, dev_r, dev_f))
test_file_logger.log('Test results for intra-sentence level: Prec:{} | Rec:{} | F1:{} '.format(intra_dev_p, intra_dev_r, intra_dev_f))
test_file_logger.log('Test results for inter-sentence level: Prec:{} | Rec:{} | F1:{} '.format(inter_dev_p, inter_dev_r, inter_dev_f))
print('Test end.....')
"""
#id2label = dict([(v,k) for k,v in label2id.items()])
dev_score_history = []
current_lr = opt['lr']
global_step = 0
global_start_time = time.time()
format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}'
def train_model(vocab_params,
train_params,
train_batch,
dev_batch,
model_id=-1):
torch.manual_seed(train_params.seed)
np.random.seed(train_params.seed)
random.seed(train_params.seed)
if train_params.cpu:
train_params.cuda = False
elif train_params.cuda:
torch.cuda.manual_seed(train_params.seed)
# make opt
opt = vars(vocab_params)
print(constant.LABEL_TO_ID)
print(opt)
opt['num_class'] = len(constant.LABEL_TO_ID)
# Combine all the parameters together
opt.update(vars(train_params))
# load vocab
vocab_file = opt['vocab_dir'] + '/vocab.pkl'
vocab = Vocab(vocab_file, load=True)
opt['vocab_size'] = vocab.size
emb_file = opt['vocab_dir'] + '/embedding.npy'
emb_matrix = np.load(emb_file)
assert emb_matrix.shape[0] == vocab.size
assert emb_matrix.shape[1] == opt['emb_dim']
if (model_id == -1):
model_id = opt['id'] if len(opt['id']) > 1 else '0' + opt['id']
model_save_dir = opt['save_dir'] + '/' + model_id
opt['model_save_dir'] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
helper.save_config(opt, model_save_dir + '/config.json', verbose=True)
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
model_save_dir + '/' + opt['log'],
header="# epoch\ttrain_loss\tdev_loss\tdev_f1")
# print model info
helper.print_config(opt)
# model
model = RelationModel(opt, emb_matrix=emb_matrix)
id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()])
dev_f1_history = []
current_lr = opt['lr']
global_step = 0
global_start_time = time.time()
format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}'
max_steps = len(train_batch) * opt['num_epoch']
# start training
for epoch in range(1, opt['num_epoch'] + 1):
train_loss = 0
for i, batch in enumerate(train_batch):
start_time = time.time()
global_step += 1
loss = model.update(batch)
train_loss += loss
if global_step % opt['log_step'] == 0:
duration = time.time() - start_time
print(format_str.format(datetime.now(), global_step, max_steps, epoch,\
opt['num_epoch'], loss, duration, current_lr))
# eval on dev
print("Evaluating on dev set...")
predictions = []
dev_loss = 0
for i, batch in enumerate(dev_batch):
preds, _, loss = model.predict(batch)
predictions += preds
dev_loss += loss
predictions = [id2label[p] for p in predictions]
dev_p, dev_r, dev_f1 = scorer.score(dev_batch.gold(), predictions)
train_loss = train_loss / train_batch.num_examples * opt[
'batch_size'] # avg loss per batch
dev_loss = dev_loss / dev_batch.num_examples * opt['batch_size']
print("epoch {}: train_loss = {:.6f}, dev_loss = {:.6f}, dev_f1 = {:.4f}".format(epoch,\
train_loss, dev_loss, dev_f1))
file_logger.log("{}\t{:.6f}\t{:.6f}\t{:.4f}".format(
epoch, train_loss, dev_loss, dev_f1))
# save
model_file = model_save_dir + '/checkpoint_epoch_{}.pt'.format(epoch)
model.save(model_file, epoch)
if epoch == 1 or dev_f1 > max(dev_f1_history):
copyfile(model_file, model_save_dir + '/best_model.pt')
print("new best model saved.")
if epoch % opt['save_epoch'] != 0:
os.remove(model_file)
# lr schedule
if len(dev_f1_history) > 10 and dev_f1 <= dev_f1_history[-1] and \
opt['optim'] in ['sgd', 'adagrad']:
current_lr *= opt['lr_decay']
model.update_lr(current_lr)
dev_f1_history += [dev_f1]
print("")
print("Training ended with {} epochs.".format(epoch))
def main():
args = get_parser()
# set seed and prepare for training
torch.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
if args.cpu:
args.cuda = False
elif args.cuda:
torch.cuda.manual_seed(args.seed)
init_time = time.time()
# make opt
opt = vars(args)
TEXT, train_batch, dev_batch = load_data(opt['batch_size'],
device='cuda:0')
vocab = TEXT.vocab
opt['vocab_size'] = len(vocab.stoi)
emb_matrix = vocab.vectors
assert emb_matrix.shape[0] == opt['vocab_size']
assert emb_matrix.shape[1] == opt['emb_dim']
model_id = opt['id'] if len(opt['id']) > 1 else '0' + opt['id']
model_save_dir = opt['save_dir'] + '/' + str(model_id)
opt['model_save_dir'] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
path = os.path.join(model_save_dir, 'config.json')
helper.save_config(opt, path, verbose=True)
# vocab.save(os.path.join(model_save_dir, 'vocab.pkl'))
file_logger = helper.FileLogger(
os.path.join(model_save_dir, opt['log']),
header="# epoch\ttrain_loss\tdev_loss\tdev_score\tbest_dev_score")
# print model info
helper.print_config(opt)
# Build Model
if not opt['load']:
trainer = LSTMTrainer(opt, emb_matrix)
else:
model_file = opt['model_file']
print("Loading model from {}".format(model_file))
model_opt = torch_utils.load_config(model_file)
model_opt['optim'] = opt['optim']
trainer = LSTMTrainer(model_opt)
trainer.load(model_file)
dev_score_history = []
current_lr = opt['lr']
global_step = 0
global_start_time = time.time()
format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}'
max_steps = len(train_batch) * opt['num_epoch']
# start training
for epoch in range(1, opt['num_epoch'] + 1):
train_loss = 0
for i, batch in enumerate(train_batch):
start_time = time.time()
global_step += 1
loss = trainer.update(batch)
train_loss += loss
if global_step % opt['log_step'] == 0:
duration = time.time() - start_time
print(format_str.format(datetime.now(), global_step, max_steps, epoch, \
opt['num_epoch'], loss, duration, current_lr))
# eval on dev
print("Evaluating on dev set ...")
predictions = []
golds = []
dev_loss = 0.0
for i, batch in enumerate(dev_batch):
preds, probs, labels, loss = trainer.predict(batch)
predictions += preds
golds += labels
dev_loss += loss
train_loss = train_loss / len(train_batch)
dev_loss = dev_loss / len(dev_batch)
# print(golds)
# print(predictions)
print(accuracy_score(golds, predictions))
dev_roc = roc_auc_score(golds, predictions)
print(
"epoch {}: train loss = {:.6f}, dev loss = {:.6f}, dev roc = {:.4f}"
.format(epoch, train_loss, dev_loss, dev_roc))
dev_score = dev_roc
file_logger.log("{}\t{:.6f}\t{:.6f}\t{:.4f}\t{:.4f}".format(
epoch, train_loss, dev_loss, dev_score,
max([dev_score] + dev_score_history)))
# save model
model_file = os.path.join(model_save_dir,
"checkpoint_epoch_{}.py".format(epoch))
trainer.save(model_file, epoch)
if epoch == 1 or dev_score > max(dev_score_history):
copyfile(model_file, model_save_dir + '/best_model.pt')
print("new best model saved.")
file_logger.log("new best model saved at epoch {}: {:.2f}"\
.format(epoch, dev_score*100))
if epoch % opt['save_epoch'] != 0:
os.remove(model_file)
if len(dev_score_history) > opt['decay_epoch'] and dev_score <= dev_score_history[-1] and \
opt['optim'] in ['sgd', 'adagrad', 'adadelta']:
current_lr *= opt['lr_decay']
trainer.update_lr(current_lr)
dev_score_history += [dev_score]
print("")
print("Training ended with {} epochs.".format(epoch))
