{
"params": [
p for n, p in param_optimizer if not any(nd in n for nd in no_decay)
],
"weight_decay": 0.001,
},
{
"params": [
p for n, p in param_optimizer if any(nd in n for nd in no_decay)
],
"weight_decay": 0.0,
},
]
num_train_steps = int(len(train_sentences) / config.TRAIN_BATCH_SIZE * config.EPOCHS)
optimizer = AdamW(optimizer_parameters, lr=3e-5)
scheduler = get_linear_schedule_with_warmup(
optimizer, num_warmup_steps=0, num_training_steps=num_train_steps
)
best_loss = np.inf
for epoch in range(config.EPOCHS):
train_loss = engine.train_fn(train_dataloader, model, optimizer, scheduler, device)
test_loss = engine.eval_fn(val_dataloader, model, device)
print(f"Train Loss = {train_loss} Valid Loss = {test_loss}")
if test_loss < best_loss:
torch.save(model.state_dict(), config.MODEL_PATH)
if SECONDARY_PATH is not None:
torch.save(model.state_dict(), SECONDARY_PATH+'model.bin')
print('saved model to : ',SECONDARY_PATH+'model.bin')
best_loss = test_loss
if tokenizer is not None:
print("success")
train_dataset = NerDataset(hp.trainset, tokenizer=tokenizer)
eval_dataset = NerDataset(hp.validset, tokenizer=tokenizer)
train_iter = data.DataLoader(dataset=train_dataset,
batch_size=hp.batch_size,
shuffle=True,
num_workers=4,
collate_fn=pad)
eval_iter = data.DataLoader(dataset=eval_dataset,
batch_size=hp.batch_size,
shuffle=False,
num_workers=4,
collate_fn=pad)
optimizer = optim.Adam(model.parameters(), lr=hp.lr)
criterion = nn.CrossEntropyLoss(ignore_index=0)
for epoch in range(1, hp.n_epochs + 1):
train(model, train_iter, optimizer, criterion)
print(f"=========eval at epoch={epoch}=========")
if not os.path.exists(hp.logdir): os.makedirs(hp.logdir)
fname = os.path.join(hp.logdir, str(epoch))
precision, recall, f1 = eval(model, eval_iter, fname)
torch.save(model.state_dict(), f"{fname}.pt")
print(f"weights were saved to {fname}.pt")
