def train(config, model):
fgm = FGM(model)
processor = TextProcessor()
label_list = processor.get_labels(config.class_path)
#加载训练数据
train_examples = processor.get_train_examples(config.train_path)
train_features = convert_examples_to_features(train_examples, label_list,
config.max_seq_length,
config.tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_ids for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
all_label_ids)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=config.batch_size,
drop_last=True)
#加载测试数据
dev_examples = processor.get_dev_examples(config.dev_path)
dev_features = convert_examples_to_features(dev_examples, label_list,
config.max_seq_length,
config.tokenizer)
all_input_ids_dev = torch.tensor([f.input_ids for f in dev_features],
dtype=torch.long)
all_input_mask_dev = torch.tensor([f.input_mask for f in dev_features],
dtype=torch.long)
all_segment_ids_dev = torch.tensor([f.segment_ids for f in dev_features],
dtype=torch.long)
all_label_ids_dev = torch.tensor([f.label_ids for f in dev_features],
dtype=torch.long)
dev_data = TensorDataset(all_input_ids_dev, all_input_mask_dev,
all_segment_ids_dev, all_label_ids_dev)
dev_sampler = SequentialSampler(dev_data)
dev_dataloader = DataLoader(dev_data,
sampler=dev_sampler,
batch_size=config.batch_size)
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = BertAdam(
optimizer_grouped_parameters,
lr=config.learning_rate,
# schedule='warmup_linear',
warmup=0.05,
t_total=config.num_epochs * len(train_dataloader))
#lookahead
# from optimizer import Lookahead
# optimizer = Lookahead(optimizer, k=5, alpha=0.5)
logger.info(f"正在使用GPU: {torch.cuda.current_device()}进行训练...")
model.train()
eval_steps = len(train_dataloader) // 2
for i in range(config.num_epochs):
total_batch = 0
eval_best_loss = float('inf')
eval_best_auc_score = float('-inf')
eval_best_acc = float('-inf')
last_improve = 0
flag = False
for step, batch in enumerate(train_dataloader):
batch = tuple(t.to(config.device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss = model(input_ids, segment_ids, input_mask, label_ids)
loss.backward()
#对抗训练
fgm.attack()
loss_adv = model(input_ids, segment_ids, input_mask, label_ids)
loss_adv.backward()
fgm.restore()
optimizer.step()
model.zero_grad()
logits = model(input_ids, segment_ids, input_mask)
logger.info(f"Epoch: {i+1}, step: {step+1}, train_loss: {loss}")
torch.save(model.state_dict(), config.save_path)
def train(config, model):
fgm = FGM(model)
processor = TextProcessor()
label_list = processor.get_labels(config.class_path)
#加载训练数据
train_examples = processor.get_train_examples(config.train_path)
train_features = convert_examples_to_features(train_examples, label_list,
config.max_seq_length,
config.tokenizer)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_ids for f in train_features],
dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
all_label_ids)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=config.batch_size,
drop_last=True)
#加载测试数据
dev_examples = processor.get_dev_examples(config.dev_path)
dev_features = convert_examples_to_features(dev_examples, label_list,
config.max_seq_length,
config.tokenizer)
all_input_ids_dev = torch.tensor([f.input_ids for f in dev_features],
dtype=torch.long)
all_input_mask_dev = torch.tensor([f.input_mask for f in dev_features],
dtype=torch.long)
all_segment_ids_dev = torch.tensor([f.segment_ids for f in dev_features],
dtype=torch.long)
all_label_ids_dev = torch.tensor([f.label_ids for f in dev_features],
dtype=torch.long)
dev_data = TensorDataset(all_input_ids_dev, all_input_mask_dev,
all_segment_ids_dev, all_label_ids_dev)
dev_sampler = SequentialSampler(dev_data)
dev_dataloader = DataLoader(dev_data,
sampler=dev_sampler,
batch_size=config.batch_size)
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=config.learning_rate,
warmup=0.05,
t_total=config.num_epochs * len(train_dataloader))
logger.info(f"正在使用GPU: {torch.cuda.current_device()}进行训练...")
model.train()
eval_steps = len(train_dataloader) // 2
for i in range(config.num_epochs):
total_batch = 0
eval_best_loss = float('inf')
eval_best_auc_score = float('-inf')
eval_best_acc = float('-inf')
last_improve = 0
flag = False
for step, batch in enumerate(train_dataloader):
batch = tuple(t.to(config.device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss = model(input_ids, segment_ids, input_mask, label_ids)
loss.backward()
# #对抗训练
fgm.attack()
loss_adv = model(input_ids, segment_ids, input_mask, label_ids)
loss_adv.backward()
fgm.restore()
optimizer.step()
model.zero_grad()
logits = model(input_ids, segment_ids, input_mask)
logger.info(f"Epoch: {i+1}, step: {step+1}, train_loss: {loss}")
if (total_batch + 1) % eval_steps == 0:
torch.save(model.state_dict(), config.save_path)
eval_accuracy, eval_precision, eval_loss, auc_score = evaluate(
config, model, dev_dataloader)
if auc_score > eval_best_auc_score:
eval_best_auc_score = auc_score
torch.save(model.state_dict(), config.save_path)
# if eval_accuracy > eval_best_acc:
# eval_best_acc = eval_accuracy
# torch.save(model.state_dict(), config.save_path)
print('*' * 80)
logger.info(
f"Epoch: {i+1}, step: {step+1}, train_loss: {loss}, eval_loss: {eval_loss}, eval_accuracy: {eval_accuracy}, eval_precision: {eval_precision}, auc_score: {auc_score}"
)
# 保存log
with open('data/match/logs.txt', 'a+',
encoding='utf-8') as log_writer:
log_writer.write(
f"Epoch: {i+1}, step: {step+1}, train_loss: {loss}, eval_loss: {eval_loss}, eval_accuracy: {eval_accuracy}, eval_precision: {eval_precision}, auc_score: {auc_score} \n"
)
model.train()
total_batch += 1