def main(args):
print(20 * "=", " Preparing for training ", 20 * "=")
if not os.path.exists(args.result):
os.makedirs(args.result)
tokenizer = tokenization.BertTokenizer(args.vocab, do_lower_case=True)
# -------------------- Data loading ------------------- #
print("\t* Loading testing data...")
# train_data = LCQMC_dataset(args.train_file, args.vocab_file, args.max_length, test_flag=False)
test_data = DataProcessForSentence(tokenizer,
args.test_file,
args.max_length,
test_flag=True)
test_loader = DataLoader(test_data, batch_size=args.batch_size)
# -------------------- Model definition ------------------- #
print("\t* Building model...")
model = Roberta_pooling(args).to(args.device)
all_predict = predict(model, test_loader, args)
index = np.array([], dtype=int)
for i in range(len(all_predict)):
index = np.append(index, i)
# ---------------------生成文件--------------------------
df_test = pd.DataFrame(columns=['index', 'prediction'])
df_test['index'] = index
df_test['prediction'] = all_predict
df_test.to_csv(args.submit_example_path,
index=False,
columns=['index', 'prediction'],
sep='\t')
def main(args):
print(20 * "=", " Preparing for training ", 20 * "=")
# 保存模型的路径
if not os.path.exists(args.target_dir):
os.makedirs(args.target_dir)
tokenizer = BertTokenizer.from_pretrained(args.vocab)
# -------------------- Data loading ------------------- #
print("\t* Loading training data...")
# train_data = LCQMC_dataset(args.train_file, args.vocab_file, args.max_length, test_flag=False)
train_data = DataProcessForSentence(tokenizer,
args.train_file,
args.max_length,
test_flag=False)
train_loader = DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True)
print("\t* Loading valid data...")
dev_data = DataProcessForSentence(tokenizer,
args.dev_file,
args.max_length,
test_flag=False)
dev_loader = DataLoader(dev_data, batch_size=args.batch_size, shuffle=True)
# -------------------- Model definition ------------------- #
print("\t* Building model...")
model = Bert_model(args).to(args.device)
# -------------------- Preparation for training ------------------- #
criterion = nn.CrossEntropyLoss() # 交叉熵损失函数
# 列出所有需要更新权重的参数
param_optimizer = list(model.named_parameters())
# 不需要权重衰减的
no_decay = ['bias', 'LearyNorm.bias', 'LayerNorm.weight']
# 不是这几种类型的就需要进行权重衰减，这里中是不需要进行权重衰减的，保持正常的梯度更新即可
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.00
}]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.lr,
warmup=0.5,
t_total=len(train_loader) * args.epochs)
# 学习计划
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='max',
factor=0.85,
patience=0)
best_score = 0.0
start_epoch = 1
epochs_count = []
train_losses = []
valid_losses = []
# Continuing training from a checkpoint if one was given as argument
if args.checkpoint:
# 从文件中加载checkpoint数据, 从而继续训练模型
checkpoints = torch.load(args.checkpoint)
start_epoch = checkpoints["epoch"] + 1
best_score = checkpoints["best_score"]
print("\t* Training will continue on existing model from epoch {}...".
format(start_epoch))
model.load_state_dict(checkpoints["model"]) # 模型部分
optimizer.load_state_dict(checkpoints["optimizer"])
epochs_count = checkpoints["epochs_count"]
train_losses = checkpoints["train_losses"]
valid_losses = checkpoints["valid_losses"]
# 这里改为只有从以前加载的checkpoint中才进行计算 valid, Compute loss and accuracy before starting (or resuming) training.
_, valid_loss, valid_accuracy, auc = validate(model, dev_loader,
criterion)
print(
"\t* Validation loss before training: {:.4f}, accuracy: {:.4f}%, auc: {:.4f}"
.format(valid_loss, (valid_accuracy * 100), auc))
# -------------------- Training epochs ------------------- #
print("\n", 20 * "=",
"Training Bert model on device: {}".format(args.device), 20 * "=")
patience_counter = 0
for epoch in range(start_epoch, args.epochs + 1):
epochs_count.append(epoch)
# -------------------- train --------------------------
print("* Training epoch {}:".format(epoch))
epoch_time, epoch_loss, epoch_accuracy = train(model, train_loader,
optimizer, criterion,
args)
train_losses.append(epoch_loss)
print("-> Training time: {:.4f}s, loss = {:.4f}, accuracy: {:.4f}%".
format(epoch_time, epoch_loss, (epoch_accuracy * 100)))
# -------------------- valid --------------------------
print("* Validation for epoch {}:".format(epoch))
epoch_time, epoch_loss, epoch_accuracy, epoch_auc = validate(
model, train_loader, criterion, args)
valid_losses.append(epoch_loss)
print(
"-> Valid. time: {:.4f}s, loss: {:.4f}, accuracy: {:.4f}%, auc: {:.4f}\n"
.format(epoch_time, epoch_loss, (epoch_accuracy * 100), epoch_auc))
# Update the optimizer's learning rate with the scheduler.
scheduler.step(epoch_accuracy)
# Early stopping on validation accuracy.
if epoch_accuracy < best_score:
patience_counter += 1
else:
best_score = epoch_accuracy
patience_counter = 0
# 保存最好的结果，需要保存的参数，这些参数在checkpoint中都能找到
torch.save(
{
"epoch": epoch,
"model": model.state_dict(),
"best_score": best_score,
"epochs_count": epochs_count,
"train_losses": train_losses,
"valid_losses": valid_losses
}, os.path.join(args.target_dir, "bert_best.bin"))
if patience_counter >= args.patience:
print("-> Early stopping: patience limit reached, stopping...")
break