project_root: Path = get_project_root()
with open(str(project_root / "config.yml")) as f:
params = yaml.load(f, Loader=yaml.FullLoader)
# read and process data
train_val_loaders, test_loaders = read_data(params)
# initialize the model
model = BertForSequenceClassification(
pretrained_model_name=params["model"]["model_name"],
num_classes=params["model"]["num_classes"],
)
# specify criterion for the multi-class classification task, optimizer and scheduler
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=float(params["training"]["learn_rate"]))
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer)
# reproducibility
set_global_seed(params["general"]["seed"])
prepare_cudnn(deterministic=True)
# here we specify that we pass masks to the runner. So model's forward method will be called with
# these arguments passed to it.
runner = SupervisedRunner(input_key=("features", "attention_mask"))
# finally, training the model with Catalyst
runner.train(
model=model,
criterion=criterion,
from transformers import get_linear_schedule_with_warmup
import time
import datetime
import os
import os.path
import random
import torch.nn as nn
import torch.optim as optim
import numpy as np
from dataloader_training import dataloader
train_dataloader = dataloader()
device = torch.device("cuda")
model = BertForSequenceClassification(3)
model.cuda()
optimizer = AdamW(model.parameters(), lr=2e-5, eps=1e-8)
epochs = 10
total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
num_training_steps=total_steps)
def format_time(elapsed):
elapsed_rounded = int(round((elapsed)))
return str(datetime.timedelta(seconds=elapsed_rounded))
def save_checkpoint(state, is_best, filename='./checkpoint_4.pth.tar'):
if is_best:
optimizer.step()
sum_loss += loss.mean().item()
sum_accuracy += accuracy(logits, label_ids)
total_examples += batch_size
total_steps += 1
result = {
"train_loss": sum_loss / total_steps,
"train_accuracy": sum_accuracy / total_examples,
}
logger.info("***** Train results *****")
for key in sorted(result.keys()):
logger.info("%s = %s", key, str(result[key]))
if __name__ == "__main__":
bert, config, vocab_file = create_hub_bert(args.pretrained_bert, pretrained=True)
args.vocab_file = vocab_file
model = BertForSequenceClassification(config, num_labels=2, bert=bert)
mrpc_dataset = MRPCDataset(args)
optimizer = optim.Adam(model.parameters(requires_grad=True), lr=args.learning_rate,)
mrpc_dataset = MRPCDataset(args)
train_dataloader, train_size = mrpc_dataset.get_train_dataloader()
eval_dataloader, eval_size = mrpc_dataset.get_eval_dataloader()
for epoch in range(args.num_train_epochs):
logger.info("***** Epoch {} *****".format(epoch + 1))
train(train_dataloader, model, optimizer)
mge.save(model.state_dict(), args.save_model_path)
eval(eval_dataloader, model)
total_steps += 1
result = {
"train_loss": sum_loss / total_steps,
"train_accuracy": sum_accuracy / total_examples,
}
logger.info("***** Train results *****")
for key in sorted(result.keys()):
logger.info("%s = %s", key, str(result[key]))
if __name__ == "__main__":
bert, config, vocab_file = create_hub_bert(args.pretrained_bert,
pretrained=True)
args.vocab_file = vocab_file
model = BertForSequenceClassification(config, num_labels=2, bert=bert)
mrpc_dataset = MRPCDataset(args)
optimizer = optim.Adam(
model.parameters(requires_grad=True),
lr=args.learning_rate,
)
mrpc_dataset = MRPCDataset(args)
train_dataloader, train_size = mrpc_dataset.get_train_dataloader()
eval_dataloader, eval_size = mrpc_dataset.get_eval_dataloader()
for epoch in range(args.num_train_epochs):
logger.info("***** Epoch {} *****".format(epoch + 1))
train(train_dataloader, model, optimizer)
mge.save(model.state_dict(), args.save_model_path)
eval(eval_dataloader, model)