def train():
"""
模型训练
"""
train_writer = SummaryWriter(log_dir='./log/train')
test_writer = SummaryWriter(log_dir='./log/test')
# step1 模型
bilstm_crf = BiLSTM_CRF(opt.vocab_size, opt.emb_dim, opt.emb_dim//2, opt.tag_num, dropout=opt.dropout)
if opt.load_model_path: # 是否加载checkpoint
bilstm_crf.load(opt.load_model_path)
# step2 数据
rmrb_train_dataset = RmrbDataset(train=True)
rmrb_test_dataset = RmrbDataset(train=False)
rmrb_train_dataloader = DataLoader(rmrb_train_dataset, batch_size=64, shuffle=True)
rmrb_test_dataloader = DataLoader(rmrb_test_dataset, batch_size=len(rmrb_test_dataset), shuffle=True)
# step3 损失函数和优化器
# loss_fn = t.nn.CrossEntropyLoss()
lr = opt.lr
optimizer = t.optim.Adam(params=bilstm_crf.parameters(), lr=lr, weight_decay=opt.weight_decay)
previous_loss = 1e9
iteration = 0
for epoch in range(opt.max_epoch):
print('epoch {}'.format(epoch))
for ii, (x_batch, y_batch) in enumerate(rmrb_train_dataloader):
# 计算loss
loss = bilstm_crf.log_likelihood(x_batch, y_batch)
loss.backward()
optimizer.step()
optimizer.zero_grad()
if ii % 20 == 0:
# print('loss:{}'.format(loss.item()))
train_writer.add_scalar('Loss', loss.item(), iteration)
iteration += 1
if loss > previous_loss:
lr = lr * opt.lr_decay
else:
previous_loss = loss.item()
# 保存模型检查点
bilstm_crf.save()
# 评价指标
with t.no_grad():
bilstm_crf.eval() # 将模型设置为验证模式
for x_test, y_test in rmrb_test_dataloader:
test_loss = bilstm_crf.log_likelihood(x_test, y_test)
test_writer.add_scalar('Loss', test_loss.item(), iteration)
y_pre = bilstm_crf(x_test)
print(classification_report(t.flatten(y_test), t.flatten(y_pre)))
bilstm_crf.train() # 将模型恢复成训练模式
def predict(sentence, print_entity=False):
"""
模型预测
"""
# 模型
bilstm_crf = BiLSTM_CRF(opt.vocab_size, opt.emb_dim, opt.emb_dim//2, opt.tag_num, dropout=opt.dropout)
if opt.load_model_path:
bilstm_crf.load(opt.load_model_path)
bilstm_crf.eval()
# 数据
x = word2idx(sentence)
x = t.LongTensor(x).unsqueeze(dim=0)
tag_idx = bilstm_crf(x).squeeze(dim=0)
tag_idx = tag_idx.numpy().tolist()
length = min(opt.max_length, len(sentence))
entity_list = []
i = 0
while i < length:
if tag_idx[i] == 1:
entity = sentence[i]
j = i + 1
for j in range(i+1, length):
if tag_idx[j] == 2:
entity += sentence[j]
else:
break
i = j
entity_list.append(entity)
else:
i += 1
if print_entity:
print(entity_list)
print('\n')
return idx2tag(tag_idx)
return sentences_sort, lengths_sort, idx_unsort
char2idx = pickle.load(open('char2idx.pkl', 'rb'))
data = pickle.load(open('predict_data.pkl', 'rb'))
predict_data = PredData(data, char2idx)
dataloader = DataLoader(predict_data, batch_size=32, drop_last=False)
model = BiLSTM_CRF(len(char2idx), len(Config.tagert2idx), Config.embedding_dim,
Config.hidden_dim)
model.load_state_dict(torch.load('model_best.pth'))
if Config.use_gpu:
model.to('cuda')
model.eval()
predict_result = []
with torch.no_grad():
for batch_sentences, batch_lengths in dataloader:
sentences, lengths, idx_unsort = sort_batch_data(
batch_sentences, batch_lengths)
if Config.use_gpu:
sentences = sentences.cuda()
pred = model(sentences, lengths)
pred = pred[idx_unsort]
pred = pred.cpu().numpy()
ls = batch_lengths.numpy()
for tags, l in zip(pred, ls):
sent_res.append([sent[i], tag[i], tag_pred[i]])
model_pred.append(sent_res)
# label_path = os.path.join(hp.result_path, 'label_' + epoch_num)
# metric_path = os.path.join(hp.result_path, 'result_metric_' + epoch_num)
result = conlleval(model_pred)
print(result)
# print(len(label_pred))
if __name__ == '__main__':
logging.info("Loading data")
train_data = load_data(hp.prepro_dir + 'train.utf8', name="train")
test_data = load_data(hp.prepro_dir + 'test.utf8', name='test')
model = BiLSTM_CRF()
loss, train_op, global_step = model.train()
logits, transition_params = model.eval()
logging.info("Graph loaded")
config_proto = tf.ConfigProto(allow_soft_placement=True)
config_proto.gpu_options.allow_growth = True
with tf.Session(config=config_proto) as sess:
saver = tf.train.Saver(max_to_keep=hp.max_to_keep)
ckpt = tf.train.latest_checkpoint(hp.log_dir)
if ckpt is None:
logging.info("Initializing from scratch")
sess.run(tf.global_variables_initializer())
else:
logging.info("Restore model from {}".format(ckpt))
saver.restore(sess, ckpt)
last_save_steps = 0
for i in range(hp.epoch):
logging.info("###### epoch {}".format(i))