def train(model, train_dataset, dev_dataset, config):
start_time = time.time()
train_dataloader = DataLoader(train_dataset,
batch_size=config.batch_size,
shuffle=True,
num_workers=2)
n_sample = len(train_dataloader)
model.train()
optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate)
classified_loss = torch.nn.CrossEntropyLoss().to(config.device)
total_batch = 0
for epoch in range(config.num_epochs):
total_loss = 0
model.train()
print('Epoch [{}/{}]'.format(epoch + 1, config.num_epochs))
for sample in tqdm(train_dataloader):
sample = (i.to(config.device) for i in sample)
sentence1, sentence2, label, seq1, seq2 = sample
# print(sentence1)
model.zero_grad()
out = model(sentence1, sentence2)
loss = classified_loss(out, label)
total_loss += loss.clone().detach()
loss.backward()
optimizer.step()
total_batch += 1
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
print(f'train loss: {total_loss / n_sample}')
evaluate(config, model, dev_dataset)
torch.save(model.state_dict(), config.save_path)
def train():
print('=========model architecture==========')
model = GCA(args).to(device)
print(model)
print('=============== end =================')
loss_fn = nn.CrossEntropyLoss(ignore_index=constants.PAD_IDX)
parameters = filter(lambda p: p.requires_grad, model.parameters())
optimizer = torch.optim.Adam(parameters, args["lr"], weight_decay=args["wdecay"])
print('Training and evaluating...')
logger.info(
"Batch Size: %d, Dropout: %.2f" % (
args["batch_size"], args["dropout"]
)
)
if args["ac_re"]:
logger.info(
"Alpha L2 Regularization: %d, Beta Slowness Regularization: %d" % (
args["alpha"], args["beta"],
)
)
start_time = time.time()
if not os.path.exists(args["exp_dir"]):
os.makedirs(args["exp_dir"])
best_ppl = 9999999
last_improved = 0
require_improvement = 5
with open(args["exp_dir"] + "params.json", "w") as outfile:
json.dump(args, outfile, indent=4)
for epoch in range(args["epochs"]):
print('=============== Epoch %d=================' % (epoch + 1))
logger.info("Optimizer: %s" % (optimizer))
train_loss, train_ppl = train_epoch(
epoch, train_dataloader, model, loss_fn, optimizer, device, args["clip"], args["ac_re"], args["alpha"],
args["beta"]
)
valid_loss, valid_ppl = test(
model, valid_dataloader, device
)
if valid_ppl < best_ppl:
best_ppl = valid_ppl
last_improved = epoch
torch.save(model.state_dict(), args["exp_dir"] +
'model_params_%s_min_ppl.pkl' % (epoch + 1)
)
improved_str = '*'
else:
improved_str = ''
time_dif = get_time_dif(start_time)
msg = 'Epoch: {0:>6},Train Loss: {1:>6.6}, Train Ppl: {2:>6.6},' \
+ ' Val loss: {3:>6.6}, Val Ppl: {4:>6.6},Time:{5} {6}'
print(msg.format(epoch + 1, train_loss, train_ppl, valid_loss, valid_ppl, time_dif, improved_str))
if epoch - last_improved > require_improvement:
print("No optimization for a long time, auto-stopping...")
break
return 1
def test(model, test_iter):
# test
model.load_state_dict(torch.load(Config.save_path))
model.eval()
start_time = time.time()
test_acc, test_loss, test_report, test_confusion = evaluate(model, test_iter, test=True)
msg = 'Test Loss: {0:>5.2}, Test Acc: {1:>6.2%}'
print(msg.format(test_loss, test_acc))
print("Precision, Recall and F1-Score...")
print(test_report)
print("Confusion Matrix...")
print(test_confusion)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
model.train()
voc.add_token(word)
for filepath in args.defs:
data = read_data(filepath)
for elem in data:
voc.add_token(elem[0])
char_voc.token_maxlen = max(len(elem[0]), char_voc.token_maxlen)
for c in elem[0]:
char_voc.add_token(c)
definition = elem[1]
for d in definition:
voc.add_token(d)
if args.usage:
usage = elem[3]
for u in usage:
voc.add_token(u)
context = elem[2]
if args.same:
for t in context:
voc.add_token(t)
else:
context_voc.add_token(elem[0])
for t in context:
context_voc.add_token(t)
voc.save(args.save)
char_voc.save(args.save_chars)
if not args.same:
context_voc.save(args.save_context)
time_dif = get_time_dif(start_time)
print("Finished! Build vocabulary time usage:", time_dif)
def train():
print('=========model architecture==========')
if args["type"] == "same_level":
model = SenseGenerator_Emb(args).to(device)
elif args["type"] == "hir_level":
model = SenseGenerator(args).to(device)
print(model)
print('=============== end =================')
loss_fn = nn.CrossEntropyLoss(ignore_index=constants.PAD_IDX)
# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
# torch.optim.Adam(
# filter(lambda p: p.requires_grad, model.parameters()), lr=args["lr"]
# ),
# factor=args["decay_factor"],
# patience=args["decay_patience"]
# )
# optimizer = scheduler.optimizer
parameters = filter(lambda p: p.requires_grad, model.parameters())
optimizer = torch.optim.Adam(parameters,
args["lr"],
weight_decay=args["wdecay"])
print('Training and evaluating...')
logger.info(
"Batch Size: %d, Dropout: %.2f, RNN Weight Dropout: %.2f, RNN Layer Dropout: %.2f, Input Embedding Layers Dropout: %.2f, "
"Remove Words Embedding Dropout: %.2f" %
(args["batch_size"], args["dropout"], args["wdrop"], args["dropouth"],
args["dropouti"], args["dropoute"]))
if args["ac_re"]:
logger.info(
"Alpha L2 Regularization: %d, Beta Slowness Regularization: %d" % (
args["alpha"],
args["beta"],
))
start_time = time.time()
if not os.path.exists(args["exp_dir"]):
os.makedirs(args["exp_dir"])
best_ppl = 9999999
last_improved = 0
require_improvement = 5
with open(args["exp_dir"] + "params.json", "w") as outfile:
json.dump(args, outfile, indent=4)
for epoch in range(args["epochs"]):
print('=============== Epoch %d=================' % (epoch + 1))
logger.info("Optimizer: %s" % (optimizer))
train_loss, train_ppl, task1_loss, task1_ppl, task2_loss, task2_ppl = train_epoch(
epoch, train_dataloader, model, loss_fn, optimizer, device,
args["clip"], args["ac_re"], args["alpha"], args["beta"])
# train_loss, train_ppl, task1_loss, task1_ppl, task2_loss, task2_ppl = train_shunxu(
# epoch, train_dataloader, model, loss_fn, optimizer, device, args["clip"]
# )
valid1_loss, valid1_ppl, valid2_loss, valid2_ppl = test(
model, args["label_type"], valid_dataloader, device)
if valid2_ppl < best_ppl:
best_ppl = valid2_ppl
last_improved = epoch
torch.save(
model.state_dict(),
args["exp_dir"] + 'hir_shared_%s_min_ppl.pkl' % (epoch + 1))
improved_str = '*'
else:
improved_str = ''
time_dif = get_time_dif(start_time)
msg = 'Epoch: {0:>6},Total Train Loss: {1:>6.6}, Total Train Ppl: {2:>6.6},Time:{3}'
print(msg.format(epoch + 1, train_loss, train_ppl, time_dif) + "\n")
task1_msg = 'Task1: {0:>6},Valid Loss: {1:>6.6}, Valid Ppl: {2:>6.6}'
task2_msg = 'Task2: {0:>6},Valid Loss: {1:>6.6}, Vliad Ppl: {2:>6.6} {3}'
print(
task1_msg.format(args["label_type"][0], valid1_loss, valid1_ppl) +
"\n")
print(
task2_msg.format(args["label_type"][1], valid2_loss, valid2_ppl,
improved_str) + "\n")
if epoch - last_improved > require_improvement:
print("No optimization for a long time, auto-stopping...")
break
return 1
