def test(test_data):
valid_loss = 0
valid_acc = 0
test_dataset = TextDataSet(test_data)
data = DataLoader(test_dataset, batch_size=args.batch_size, collate_fn=generate_batch)
for i, (label, inp) in enumerate(data):
with torch.no_grad():
output = model(inp)
loss = criterion(output, label)
valid_loss += loss.item()
valid_acc += (output.argmax(1) == label).sum().item()
return valid_loss / len(test_dataset), valid_acc / len(test_dataset)
def train(train_data):
train_loss = 0
train_acc = 0
train_dataset = TextDataSet(train_data)
data = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, collate_fn=generate_batch)
for i, (label, inp) in enumerate(data):
optimizer.zero_grad()
output = model(inp)
loss = criterion(output, label)
train_loss += loss.item()
loss.backward()
optimizer.step()
train_acc += (output.argmax(1) == label).sum().item()
return train_loss / len(train_dataset), train_acc / len(train_dataset)
def train():
global model
# calc total steps before set scheduler
scheduler_dataset = TextDataSet(config.train)
scheduler_dataloader = DataLoader(scheduler_dataset,
batch_size=config.batch_size,
shuffle=False,
collate_fn=generate_batch)
total_steps = config.epoch * len(scheduler_dataloader)
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
num_training_steps=total_steps)
best_valid_acc = 0.0
model.train()
for epoch in range(config.epoch):
start_time = time.time()
train_loss = 0
train_acc = 0
train_count = 0
train_dataset = TextDataSet(config.train)
data = DataLoader(train_dataset,
batch_size=config.batch_size,
shuffle=True,
collate_fn=generate_batch)
for i, (label, input_ids, attention_masks) in enumerate(data):
optimizer.zero_grad()
output = model(input_ids, attention_masks)
loss = criterion(output, label)
train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
scheduler.step()
train_acc += (output.argmax(1) == label).sum().item()
train_count += input_ids.shape[0]
step_accuracy = train_acc / train_count
if i % config.save_step == 0:
print("Epoch:%s Steps:%s train_count:%s" %
(epoch, i, train_count))
valid_loss, valid_acc = test(config.dev)
if valid_acc > best_valid_acc and epoch > 0:
save_model()
best_valid_acc = valid_acc
secs = int(time.time() - start_time)
mins = secs / 60
secs = mins % 60
writer.add_scalars("Loss", {
'train': train_loss / train_count,
"valid": valid_loss
}, epoch * total_steps + i)
writer.add_scalars("Acc", {
'train': train_acc / train_count,
"valid": valid_acc
}, epoch * total_steps + i)
print("Epoch: %d" % (epoch + 1),
"Steps: %d" % (epoch * total_steps + i),
" | time in %d minutes, %d seconds" % (mins, secs))
print(
f"\tLoss: {train_loss / train_count:.4f}(train)\t|\tAcc: {train_acc / train_count * 100:.1f}%(train)"
)
print(
f"\tLoss: {valid_loss:.4f}(valid)\t|\tAcc: {valid_acc * 100:.1f}%(valid)"
)
# test
model_path = "model/pytorch_model.bin"
model = BertPretrainClassification.from_pretrained(model_path,
config=bert_config)
model.to(device)
test_loss, test_acc = test(config.test)
print(
f"\tLoss: {test_loss:.4f}(test)\t|\tAcc: {test_acc * 100:.1f}%(test)")