def test(config, model, test_iter, use_type):
# test
model.load_state_dict(torch.load(config.save_path))
model.eval()
start_time = time.time()
if use_type == 'train':
test_acc, test_loss, test_report = evaluate(model, test_iter, test=True) # , test_confusion
msg = 'Test Loss: {0:>5.2}, Test Acc: {1:>6.2%}'
print(msg.format(test_loss, test_acc))
print("Precision, Recall and F1-Score...")
print(test_report)
# print("Confusion Matrix...")
# print(test_confusion)
else:
predict_all = np.array([], dtype=int)
with torch.no_grad():
for texts, labels in test_iter:
outputs = model(texts)
pred = torch.max(outputs.data, 1)[1].cpu().numpy()
predict_all = np.append(predict_all, pred)
# 输出DevTag
with open(path.user_test_result_path, 'w', encoding='utf-8') as f:
for each in predict_all:
device_type, brand, product = config.class_list[each].split('/')
devtag = {
'vendor': brand,
'product': product,
'device_type': device_type,
}
json.dump(devtag, f)
f.write('\n')
print("Save DevTag to ", path.user_test_result_path)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
def test(config, model, test_iter):
# test
model.load_state_dict(torch.load(config.save_path))
model.eval()
start_time = time.time()
test_acc, test_loss, test_report, test_confusion, features, labels, weight_all = evaluate(
config, model, test_iter, test=True)
print(features.shape)
msg = 'Test Loss: {0:>5.2}, Test Acc: {1:>6.2%}'
print(msg.format(test_loss, test_acc))
print("Precision, Recall and F1-Score...")
print(test_report)
print("Confusion Matrix...")
print(test_confusion)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
print("Write sentence word weight to file...")
all_weight_data_str = '\n'.join(weight_all)
f = open('sentence_pattern_weight.txt', 'w', encoding='utf-8')
f.write(all_weight_data_str)
f.close()
tsne = TSNE(n_components=2, init='pca', random_state=0)
t0 = time.time()
print('T-SNE feature...')
result = tsne.fit_transform(features)
print("Starting Drawing Features Classification...")
pdf = PdfPages('ohsumed_gcn_doc_test_2nd_layer.pdf')
plt.scatter(result[:, 0], result[:, 1], c=labels, cmap='Set1')
plt.tight_layout()
pdf.savefig()
plt.show()
pdf.close()
def test(config, model, test_iter):
"""
模型测试
:param config:
:param model:
:param test_iter:
:return:
"""
model.load_state_dict(torch.load(config.save_path)) # 模型加载
model.eval() # 评估模式，不启用 BatchNormalization 和 Dropout
start_time = time.time()
test_acc, test_loss, test_report, test_confusion = evaluate(config,
model,
test_iter,
test=True)
msg = 'Test Loss: {0:>5.2}, Test Acc: {1:>6.2%}'
print(msg.format(test_loss, test_acc))
print('Precision, Recall and F1-Score')
print(test_report)
print('Confusion Matrix')
print(test_confusion)
time_idf = utils.get_time_dif(start_time)
print('使用时间: ', time_idf)
def test(config, model, test_iter):
# test
model.load_state_dict(torch.load(config.save_path))
model.eval()
start_time = time.time()
# test_acc, test_loss, test_report, test_confusion, wrong_list, wrong_number, wrong_rate1 = evaluate(config, models, test_iter, test=True)
test_acc, test_loss, wrong_list, wrong_number, wrong_rate1, test_report = evaluate(
config, model, test_iter, test=True)
msg = 'Test Loss: {0:>5.2}, Test Acc: {1:>6.2%}'
print(msg.format(test_loss, test_acc))
print("Precision, Recall and F1-Score...")
print(test_report)
# print("Confusion Matrix...")
# print(test_confusion)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
return str(test_loss.cpu().item()), str(test_acc)
print('----------------------------------')
print('pre, lab, pre_galab, galab')
wrong_dict = {}
for line in wrong_list:
if line[1] not in wrong_dict.keys():
wrong_dict[line[1]] = 1
else:
wrong_dict[line[1]] += 1
print(line)
for key, value in wrong_dict.items():
print(key + ' ' + str(value) + '条')
print('test number 180300')
print('小税号预测错误条数 {}'.format(len(wrong_list)))
print('在预测错误的税号中，小税号不在正确大税号下的概率{}%'.format(wrong_rate1 * 100))
def train(config, model, train_iter, dev_iter, test_iter):
start_time = time.time()
# EMA初始化
ema = EMA(model, 0.999)
ema.register()
model.train()
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
# optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate)
optimizer = BertAdam(optimizer_grouped_parameters,
lr=config.learning_rate,
warmup=0.05,
t_total=len(train_iter) * config.num_epochs)
total_batch = 0 # 记录进行到多少batch
dev_best_loss = float('inf')
last_improve = 0 # 记录上次验证集loss下降的batch数
flag = False # 记录是否很久没有效果提升
model.train()
for epoch in range(config.num_epochs):
print('Epoch [{}/{}]'.format(epoch + 1, config.num_epochs))
for i, (trains, labels) in enumerate(train_iter):
outputs = model(trains)
model.zero_grad()
loss = F.cross_entropy(outputs, labels)
loss.backward()
optimizer.step()
ema.update() # 训练过程中，更新完参数后，同步update shadow weights
if total_batch % 100 == 0:
# 每多少轮输出在训练集和验证集上的效果
true = labels.data.cpu()
predic = torch.max(outputs.data, 1)[1].cpu()
train_acc = metrics.accuracy_score(true, predic)
ema.apply_shadow() # model应用shadow weights，进行验证、保存模型
dev_acc, dev_loss = evaluate(config, model, dev_iter)
if dev_loss < dev_best_loss:
dev_best_loss = dev_loss
torch.save(model.state_dict(), config.save_path)
improve = '*'
last_improve = total_batch
else:
improve = ''
ema.restore() # 下一次训练之前，恢复模型参数
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>5.2}, Train Acc: {2:>6.2%}, Val Loss: {3:>5.2}, Val Acc: {4:>6.2%}, Time: {5} {6}'
print(msg.format(total_batch, loss.item(), train_acc, dev_loss, dev_acc, time_dif, improve))
model.train()
total_batch += 1
if total_batch - last_improve > config.require_improvement:
# 验证集loss超过1000batch没下降，结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break
if flag:
break
test(config, model, test_iter)
def wunaijiade(self):
dataset = 'THUCNews' # 数据集
model_name = args.model # bert
x = import_module('models.' + model_name)
config = x.Config(dataset)
np.random.seed(1)
torch.manual_seed(1)
torch.cuda.manual_seed_all(1)
torch.backends.cudnn.deterministic = True # 保证每次结果一样
start_time = time.time()
print("Loading data...")
test_data = build_dataset(config)
# test_data = build_dataset(config)
test_iter = build_iterator(test_data, config)
# print("test_data",test_data)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
# train
model = x.Model(config) # .to(config.device)
# model = x.Model(config)
# beipoyuanli.train(config, model, train_iter, dev_iter, test_iter)
aa = train(config, model, test_iter)
# train(config, model, test_iter)
return aa
def test(config, model, test_iter):
# test
model.load_state_dict(torch.load(config.save_path))
model.eval()
start_time = time.time()
test_acc, test_loss, test_report, test_confusion = evaluate(config,
model,
test_iter,
test=True)
msg = 'Test Loss: {0:>5.2}, Test Acc: {1:>6.2%}'
print(msg.format(test_loss, test_acc))
print("Precision, Recall and F1-Score...")
print(test_report)
print("Confusion Matrix...")
print(test_confusion)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
with open('./real/data/confusion.txt', 'w') as fout:
test_confusion = test_confusion.tolist()
from predict import label2num
labels_name = []
for x in label2num:
labels_name.append(x)
fout.write('\t' + str(x))
fout.write('\n')
cnt = 0
for x in test_confusion:
fout.write(labels_name[cnt])
cnt += 1
fout.write('\t' + str(x))
fout.write('\n')
def test(config, model, test_iter):
# test
checkpoint = torch.load(config.save_path)
model.load_state_dict(checkpoint['model_state_dict'])
model.eval()
start_time = time.time()
test_acc, test_loss, test_report, test_confusion = evaluate(config,
model,
test_iter,
test=True)
msg = 'Test Loss: {0:>5.2}, Test Acc: {1:>6.2%}'
print(msg.format(test_loss, test_acc))
print("Precision, Recall and F1-Score...")
print(test_report)
print("Confusion Matrix...")
print(test_confusion)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
with open(config.save_dic + 'report.txt', 'w') as f:
f.write(msg.format(test_loss, test_acc))
f.write('\n')
f.write("Precision, Recall and F1-Score...")
f.write(str(test_report))
f.write('\n')
f.write("Confusion Matrix...\n")
f.write(str(test_confusion))
def test(config, model, test_iter):
"""
模型测试
:param config:
:param model:
:param test_iter:
:return:
"""
model.load_state_dict(torch.load(config.save_path))
model.eval()
start_time = time.time()
#hook降维后的数据(128,768)
# features_in_hook = torch.zeros((128,10),device='cuda:0')
#
# def hook(module, fea_in, fea_out):
# nonlocal features_in_hook
# # fea_in[0]是因为中间层输出变成了只有一个张量的元组，把张量取出来，我也不知道为什么是元组
# features_in_hook = torch.cat((features_in_hook,fea_in[0]),0)
# return None
#
# model.hooklayer.register_forward_hook(hook)
#hook句子的词嵌入向量打平后的结果
# features_in_hook = torch.zeros((128, 32*768), device='cuda:0')
#
# def hook(module, fea_in, fea_out):
# nonlocal features_in_hook
# # fea_in[0]是因为中间层输出变成了只有一个张量的元组，把张量取出来，我也不知道为什么是元组
# features_in_hook = torch.cat((features_in_hook, fea_in[0]), 0)
# return None
#
# model.f1.register_forward_hook(hook)
#hook最终输出结果（送入softmax之前的数据）
# features_in_hook = torch.zeros((128,10),device='cuda:0')
#
# def hook(module, fea_in, fea_out):
# nonlocal features_in_hook
# # 当hook的输出时，类型为张量，不再是装着张量的元组
# features_in_hook = torch.cat((features_in_hook,fea_out),0)
# return None
#
# model.hooklayer.register_forward_hook(hook)
test_acc, test_loss, test_report, test_confusion = evaluate(config,
model,
test_iter,
test=True)
# torch.save(features_in_hook, config.model_name+'10.pkl')
msg = 'Test Loss:{0:>5.2}, Test Acc:{1:>6.2%}'
print(msg.format(test_loss, test_acc))
print("Precision, Recall and F1-Score")
print(test_report)
print("Confusion Maxtrix")
print(test_confusion)
time_dif = utils.get_time_dif(start_time)
print("使用时间：", time_dif)
def Test(model_name):
dataset = 'C:/Users/USER/Documents/Capstone_Project/datalogs' # 数据集
p = os.path.dirname(os.path.dirname((os.path.abspath(__file__))))
if p not in sys.path:
sys.path.append(p)
do_train = False
do_test = True
x = import_module('models.{}'.format(model_name))
config = x.Config(dataset)
np.random.seed(156)
torch.cuda.manual_seed_all(1024)
torch.backends.cudnn.deterministic = True
start_time = time.time()
print('Loading data...')
train_data, dev_data, test_data = build_dataset(config, do_train, do_test)
if do_test:
test_iter = build_iterator(test_data, config, do_dev=True)
time_dif = get_time_dif(start_time)
model = x.Seq2SeqModel(config).to(config.device)
for p in model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
if do_test:
test(config, model, test_iter)
def train(config, model, train_iter, dev_iter, test_iter):
start_time = time.time()
model.train()
optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate)
# 学习率指数衰减，每次epoch：学习率 = gamma * 学习率
# scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
total_batch = 0 # 记录进行到多少batch
dev_best_loss = float('inf')
last_improve = 0 # 记录上次验证集loss下降的batch数
flag = False # 记录是否很久没有效果提升
writer = SummaryWriter(log_dir=config.log_path + '/' +
time.strftime('%m-%d_%H.%M', time.localtime()))
for epoch in range(config.num_epochs):
print('Epoch [{}/{}]'.format(epoch + 1, config.num_epochs))
# scheduler.step() # 学习率衰减
for i, (trains, labels) in enumerate(train_iter):
if trains[0].shape[0] == 0:
break
# trains = trains[0].view(trains[0].shape[0], config.pad_size)
# time.sleep(100000)
optimizer.zero_grad()
outputs = model(trains)
loss = F.cross_entropy(outputs, labels)
# print(type(outputs), type(outputs[0]), outputs[0].shape, outputs[1].shape)
loss.backward()
optimizer.step()
if total_batch % 100 == 0:
# 每多少轮输出在训练集和验证集上的效果
true = labels.data.cpu()
predic = torch.max(outputs.data, 1)[1].cpu()
train_acc = metrics.accuracy_score(true, predic)
dev_acc, dev_loss = evaluate(config, model, dev_iter)
if dev_loss < dev_best_loss:
dev_best_loss = dev_loss
torch.save(model.state_dict(), config.save_path)
improve = '*'
last_improve = total_batch
else:
improve = ''
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>5.2}, Train Acc: {2:>6.2%}, Val Loss: {3:>5.2}, Val Acc: {4:>6.2%}, Time: {5} {6}'
print(
msg.format(total_batch, loss.item(), train_acc, dev_loss,
dev_acc, time_dif, improve))
writer.add_scalar("loss/train", loss.item(), total_batch)
writer.add_scalar("loss/dev", dev_loss, total_batch)
writer.add_scalar("acc/train", train_acc, total_batch)
writer.add_scalar("acc/dev", dev_acc, total_batch)
model.train()
total_batch += 1
if total_batch - last_improve > config.require_improvement:
# 验证集loss超过1000batch没下降，结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break
if flag:
break
writer.close()
predict(model, test_iter)
def train(train_iter, dev_iter, test_iter, model, epochs, learning_rate):
np.random.seed(1)
torch.manual_seed(1)
torch.cuda.manual_seed_all(1)
torch.backends.cudnn.deterministic = True
print(model.parameters)
init_network(model)
model.train()
start_time = time.time()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
total_batch = 0
last_improve = 0
writer = SummaryWriter(
log_dir=time.strftime('%m-%d_%H.%M', time.localtime()))
dev_best_loss = float('inf')
flag = False
for epoch in range(epochs):
for train_batch, label_batch in train_iter:
train_batch, label_batch = Variable(train_batch).to(
device), Variable(label_batch).to(device)
# import ipdb; ipdb.set_trace()
outputs = model(train_batch)
model.zero_grad()
loss = F.cross_entropy(outputs, label_batch)
loss.backward()
optimizer.step()
if total_batch % 100 == 0:
true = label_batch.data.cpu()
predic = torch.max(outputs.data, 1)[1].cpu()
train_acc = metrics.accuracy_score(true, predic)
dev_acc, dev_loss = validation(model, dev_iter)
if dev_loss < dev_best_loss:
dev_best_loss = dev_loss
torch.save(model.state_dict(), save_path)
improve = '*'
last_improve = total_batch
else:
improve = ''
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>5.2}, Train Acc: {2:>6.2%}, Val Loss: {3:>5.2}, Val Acc: {4:>6.2%}, Time: {5} {6}'
print(
msg.format(total_batch, loss.item(), train_acc, dev_loss,
dev_acc, time_dif, improve))
writer.add_scalar("loss/train", loss.item(), total_batch)
writer.add_scalar("loss/dev", dev_loss, total_batch)
writer.add_scalar("acc/train", train_acc, total_batch)
writer.add_scalar("acc/dev", dev_acc, total_batch)
model.train()
total_batch += 1
if total_batch - last_improve > 1000:
# 验证集loss超过1000batch没下降，结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break
if flag:
break
writer.close()
test(model, test_iter)
def train(config, model, train_iter, dev_iter, test_iter):
start_time = time.time()
model.train() # 设置模型为训练模式，会使用BN和dropout
optimizer = torch.optim.Adam(params=model.parameters(),
lr=config.learning_rate)
# 学习率指数衰减，每次epoch：学习率 = gamma * 学习率
# scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
total_batch = 0 # 记录进行到多少batch
dev_best_loss = float('inf') # 正无穷
last_improve = 0 # 记录上次验证集loss下降的batch数
flag = False # 记录是否很久没有效果提升
writer = SummaryWriter(log_dir=config.log_path + '/' +
time.strftime('%m-%d_%H.%M', time.localtime()))
for epoch in range(config.num_epochs):
print('Epoch [{}/{}]'.format(epoch + 1, config.num_epochs))
# scheduler.step() # 学习率衰减
for i, (trains, labels) in enumerate(train_iter):
outputs = model(trains)
model.zero_grad(
) # 模型中参数的梯度设为0，根据pytorch中的backward()函数的计算，当网络参量进行反馈时，梯度是被积累的而不是被替换掉；但是在每一个batch时毫无疑问并不需要将两个batch的梯度混合起来累积，因此这里就需要每个batch设置一遍zero_grad 了。
loss = F.cross_entropy(outputs, labels) # 计算损失
loss.backward() # 反向传播计算梯度
optimizer.step() # 更新模型参数
if total_batch % 100 == 0:
# 每多少轮输出在训练集和验证集上的效果
true = labels.data.cpu()
predic = torch.max(outputs.data, 1)[1].cpu()
train_acc = metrics.accuracy_score(true, predic)
dev_acc, dev_loss = evaluate(config, model, dev_iter)
if dev_loss < dev_best_loss:
dev_best_loss = dev_loss
torch.save(model.state_dict(), config.save_path
) # 只保存神经网络的训练模型参数，save的对象是net.state_dict()
improve = '*'
last_improve = total_batch
else:
improve = ''
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>5.2}, Train Acc: {2:>6.2%}, Val Loss: {3:>5.2}, Val Acc: {4:>6.2%}, Time: {5} {6}'
print(
msg.format(total_batch, loss.item(), train_acc, dev_loss,
dev_acc, time_dif, improve))
writer.add_scalar("loss/train", loss.item(), total_batch)
writer.add_scalar("loss/dev", dev_loss, total_batch)
writer.add_scalar("acc/train", train_acc, total_batch)
writer.add_scalar("acc/dev", dev_acc, total_batch)
model.train() # 由于在验证集上进行了EVAL，所以又要重新将model设为train，模式
total_batch += 1
if total_batch - last_improve > config.require_improvement:
# 验证集loss超过1000batch没下降，结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break
if flag:
break
writer.close()
test(config, model, test_iter)
def test(config,model,test_iter1,test_iter2,test_iter3):
model.load_state_dict(torch.load(config.save_path))
model.eval()
start_time=time.time()
predict(config, model, test_iter1,"../prediction_result/ocnli_predict.csv",1)
predict(config, model, test_iter2, "../prediction_result/tnews_predict.csv",2)
predict(config, model, test_iter3, "../prediction_result/ocemotion_predict.csv",3)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
def train(config, model, train_iter, dev_iter, test_iter):
'''
:param config: 模型参数配置
:param model: 模型
:param train_iter: 训练集迭代器
:param dev_iter: 开发集迭代器
:param test_iter: 测试集迭代器
:return:
'''
start_time = time.time()
model.train() # 训练模式
optimizer = torch.optim.Adam(model.parameters(),
lr=config.learning_rate) # 优化器
total_batch = 0 # 记录进行到多少batch
dev_best_loss = float('inf')
last_improve = 0 # 记录上次验证集loss下降的batch数
flag = False # 记录是否很久没有效果提升
for epoch in range(config.num_epochs):
print('Epoch [{}/{}]'.format(epoch + 1, config.num_epochs))
for i, (trains, labels) in enumerate(train_iter):
outputs = model(trains)
# 计算loss
loss = F.cross_entropy(outputs, labels)
# 反向传播
model.zero_grad()
loss.backward()
optimizer.step()
if total_batch % 100 == 0:
# 每100个batch输出在训练集和开发集上的效果
true = labels.data.cpu().tolist()
predict = torch.max(outputs.data, 1)[1].cpu().tolist()
train_acc = metrics.accuracy_score(true, predict)
dev_acc, dev_loss = evaluate(config, model, dev_iter)
if dev_loss < dev_best_loss:
dev_best_loss = dev_loss
torch.save(model.state_dict(),
config.save_path) # 保存当前模型参数
improve = '*' # *代表性能有提升
last_improve = total_batch
else:
improve = '' # 性能没有提升
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>5.2}, Train Acc: {2:>6.2%}, Val Loss: {3:>5.2}, Val Acc: {4:>6.2%}, Time: {5} {6}'
print(
msg.format(total_batch, loss.item(), train_acc, dev_loss,
dev_acc, time_dif, improve))
model.train()
total_batch += 1
if total_batch - last_improve > config.require_improvement:
# 开发集loss超过1000batch没下降，结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break
if flag:
break
test(config, model, test_iter)
def train(config, model, train_iter, dev_iter, test_iter):
start_time = time.time()
model.train()
optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate)
# scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
total_batch = 0 # batch number
dev_best_loss = float('inf')
last_improve = 0
flag = False # improve ture or false
writer = SummaryWriter(log_dir=config.log_path + '/' +
time.strftime('%m-%d_%H.%M', time.localtime()))
for epoch in range(config.num_epochs):
print('Epoch [{}/{}]'.format(epoch + 1, config.num_epochs))
# scheduler.step()
for i, (trains, labels) in enumerate(train_iter):
outputs = model(trains)
model.zero_grad()
#loss = F.cross_entropy(outputs, labels)
focalloss = MultiCEFocalLoss()
loss = focalloss(outputs, labels)
#print('training loss')
#print(loss)
loss.backward()
optimizer.step()
if total_batch % 100 == 0:
true = labels.data.cpu()
predic = torch.max(outputs.data, 1)[1].cpu()
train_acc = metrics.accuracy_score(true, predic)
dev_acc, dev_loss = evaluate(config, model, dev_iter)
if dev_loss < dev_best_loss:
dev_best_loss = dev_loss
torch.save(model.state_dict(), config.save_path)
improve = '*'
last_improve = total_batch
else:
improve = ''
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>5.2}, Train Acc: {2:>6.2%}, Val Loss: {3:>5.2}, Val Acc: {4:>6.2%}, Time: {5} {6}'
print(
msg.format(total_batch, loss.item(), train_acc, dev_loss,
dev_acc, time_dif, improve))
writer.add_scalar("loss/train", loss.item(), total_batch)
writer.add_scalar("loss/dev", dev_loss, total_batch)
writer.add_scalar("acc/train", train_acc, total_batch)
writer.add_scalar("acc/dev", dev_acc, total_batch)
model.train()
total_batch += 1
if total_batch - last_improve > config.require_improvement:
print("No optimization for a long time, auto-stopping...")
flag = True
break
if flag:
break
writer.close()
test(config, model, test_iter)
def predction(config, model, test_Iter):
"""
模型测试
"""
model.load_state_dict(torch.load(config.save_path))
model.eval()
start_time = time.time()
loss, acc = evaluation(config, model, test_Iter)
time_dif = utils.get_time_dif(start_time)
print("使用时间：", time_dif)
print('loss:{:.4f}, acc:{:.4f}'.format(loss, acc))
def train(self, config, model, train_iter, dev_iter, test_iter):
start_time = time.time()
model.train()
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]
# optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate)
optimizer = BertAdam(optimizer_grouped_parameters,
lr=config.learning_rate,
warmup=0.05,
t_total=len(train_iter) * config.num_epochs)
total_batch = 0 # 记录进行到多少batch
dev_best_loss = float('inf')
last_improve = 0 # 记录上次验证集loss下降的batch数
flag = False # 记录是否很久没有效果提升
model.train()
for epoch in range(config.num_epochs):
print('Epoch [{}/{}]'.format(epoch + 1, config.num_epochs))
for i, (trains, labels) in enumerate(train_iter):
predicts, predicts_list = model(trains, labels, mode="train")
losses = list(map(self._get_loss, predicts, labels))
loss = torch.mean(torch.stack(losses))
model.zero_grad()
#loss = F.cross_entropy(losses, labels)
loss.backward()
optimizer.step()
if total_batch % 1000 == 0:
dev_acc_lev1, dev_acc_lev2, dev_loss = self.evaluate(config, model, dev_iter)
if dev_loss < dev_best_loss:
dev_best_loss = dev_loss
torch.save(model.state_dict(), config.save_path)
improve = '*'
last_improve = total_batch
else:
improve = ''
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>5.2}, Val Loss: {2:>5.2}, Val_lev1 Acc: {3:>6.2%}, Val_lev2 Acc: {4:>6.2%}, Time: {5} {6}'
print(msg.format(total_batch, loss.item(), dev_loss, dev_acc_lev1, dev_acc_lev2, time_dif, improve))
model.train()
if total_batch > 10 and total_batch % 90000 == 0:
self.test(config, model, train_iter)
total_batch += 1
if total_batch - last_improve > config.require_improvement:
# 验证集loss超过1000batch没下降，结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break
if flag:
break
self.test(config, model, test_iter)
def baseline_method(x, config, dataset):
# 准备预测数据集
start_time = time.time()
predict_model = Predict_Baseline(dataset=dataset, config=config)
predict_model.build_dataset(path=dataset + '/data/test_data.csv')
time_dif = get_time_dif(start_time)
print('Time usage:', time_dif)
# 预测并写入文件
model = x.Model(config).to(config.device)
predict_labels = predict_model.predict(model=model)
predict_model.write_csv(labels=predict_labels,
path=dataset + '/data/result_baseline.csv')
def predict(self, model):
config = self.config
model.load_state_dict(torch.load(config.save_path))
model.eval()
start_time = time.time()
print('prediction ...')
predict_labels, ids = self.evaluate(model)
time_dif = get_time_dif(start_time)
print('Done !')
print('prediction usage:', time_dif)
return predict_labels, ids
def train(config, model, train_iter, dev_iter):
start_time = time.time()
model.train()
optimizer = AdamW(model.parameters(),
lr=config.learning_rate,
weight_decay=1e-2)
total_batch = 0 # 记录进行到多少batch
# dev_best_loss = float('inf')
dev_best_acc = -float('inf')
last_improve = 0 # 记录上次验证集loss下降的batch数
flag = False # 记录是否很久没有效果提升
model.train()
for epoch in range(config.num_epochs):
print('Epoch [{}/{}]'.format(epoch + 1, config.num_epochs))
for i, batch in enumerate(train_iter):
batch = tuple(t.to(config.device) for t in batch)
trains = (batch[0], batch[1], batch[2])
labels = batch[3]
outputs = model(trains)
model.zero_grad()
loss = F.cross_entropy(outputs, labels)
loss.backward()
optimizer.step()
if total_batch % 100 == 0:
# 每多少轮输出在训练集和验证集上的效果
true = labels.data.cpu()
predic = torch.max(outputs.data, 1)[1].cpu()
train_acc = metrics.accuracy_score(true, predic)
dev_acc, dev_loss, ouputs_all = evaluate(
config, model, dev_iter)
if dev_acc > dev_best_acc:
# dev_best_loss = dev_loss
dev_best_acc = dev_acc
torch.save(model.state_dict(), config.save_path + '.bin')
improve = '*'
last_improve = total_batch
else:
improve = ''
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>5.2}, Train Acc: {2:>6.2%}, Val Loss: {3:>5.2}, Val Acc: {4:>6.2%}, Time: {5} {6}'
print(
msg.format(total_batch, loss.item(), train_acc, dev_loss,
dev_acc, time_dif, improve))
model.train()
total_batch += 1
if total_batch - last_improve > config.require_improvement:
# 验证集loss超过1000batch没下降，结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break
if flag:
break
def predict(config, model, test_iter):
model.load_state_dict(torch.load(config.save_path))
model.eval()
start_time = time.time()
predict_all = np.array([], dtype=int)
with torch.no_grad():
for texts, labels in test_iter:
outputs = model(texts)
predic = torch.max(outputs.data, 1)[1].cpu().numpy()
predict_all = np.append(predict_all, predic)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
return predict_all
def teacher_train(model, config, train_loader, test_loader):
start_time = time.time()
model.train()
optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate)
# model.load_state_dict(torch.load('data/saved_dict/xlnet.ckpt'))
total_batch = 0 # 记录进行到多少batch
dev_best_loss = float('inf')
last_improve = 0 # 记录上次验证集loss下降的batch数
flag = False # 记录是否很久没有效果提升
model.train()
for epoch in range(config.teacher_num_epochs):
print('Epoch [{}/{}]'.format(epoch + 1, config.teacher_num_epochs))
for i, (ids, mask , labels) in enumerate(train_loader):
# print(total_batch)
ids = ids.to(config.device)
mask = mask.to(config.device)
labels = labels.to(config.device)
outputs = model(ids, mask)
model.zero_grad()
loss = F.cross_entropy(outputs, labels)
loss.backward()
optimizer.step()
# for name, w in model.named_parameters():
# if w.requires_grad:
# print(name)
if total_batch % 10 == 0:
# 每多少轮输出在训练集和验证集上的效果
true = labels.data.cpu()
predic = torch.max(outputs.data, 1)[1].cpu()
train_acc = metrics.accuracy_score(true, predic)
dev_acc, dev_loss = teacher_evaluate(model, config, test_loader)
if dev_loss < dev_best_loss:
dev_best_loss = dev_loss
torch.save(model.state_dict(), config.teacher_save_path)
improve = '*'
last_improve = total_batch
else:
improve = ''
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>5.2}, Train Acc: {2:>6.2%}, Val Loss: {3:>5.2}, Val Acc: {4:>6.2%}, Time: {5} {6}'
print(msg.format(total_batch, loss.item(), train_acc, dev_loss, dev_acc, time_dif, improve))
model.train()
total_batch += 1
if total_batch - last_improve > config.require_improvement:
# 验证集loss超过1000batch没下降，结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break
if flag:
break
teacher_test(model, config, test_loader)
def test(config, model, test_iter):
# test
model.load_state_dict(torch.load(config.save_path))
model.eval()
start_time = time.time()
test_acc, test_loss, test_report, test_confusion = evaluate(config, model, test_iter, test=True)
msg = 'Test Loss: {0:>5.2}, Test Acc: {1:>6.2%}'
print(msg.format(test_loss, test_acc))
print("Precision, Recall and F1-Score...")
print(test_report)
print("Confusion Matrix...")
print(test_confusion)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
def load_dataset(self, path, pad_size):
contents = []
config = self.config
# 篇章切割
print('cut paras ...')
start_time = time.time()
with open(path, 'r', encoding='utf-8') as fin:
cnt = 0
data = []
for line in tqdm(fin):
lin = line.strip()
if not line:
continue
cnt += 1
if cnt == 1:
continue
pos = lin.find(',')
id = lin[:pos]
content = lin[pos + 1:]
paras = cut_para_many_times(content)
for para in paras:
#if len(para) < min_length:
# continue
data.append((int(id), para))
print('Done!')
print('\nparas:', len(data))
print('Time usage:', get_time_dif(start_time))
print('\n Getting tokens ...')
for id, content in tqdm(data):
token = config.tokenizer.tokenize(content)
token = [CLS] + token
seq_len = len(token)
mask = []
token_ids = config.tokenizer.convert_tokens_to_ids(token)
if pad_size:
if len(token) < pad_size:
mask = [1] * len(token_ids) + [0] * (pad_size -
len(token))
token_ids += ([0] * (pad_size - len(token)))
else:
mask = [1] * pad_size
token_ids = token_ids[:pad_size]
seq_len = pad_size
contents.append((token_ids, int(id), seq_len, mask))
# print('\nlen(contents) : ', str(len(contents))+'\n')
return contents
def final_predict(config, model, data_iter):
model.load_state_dict(torch.load(config.save_path))
model.eval()
start_time = time.time()
predict_final = []
predict_final = np.array([], dtype=int)
with torch.no_grad():
for texts, labels in data_iter:
outputs = model(texts)
pred = torch.max(outputs.data, 1)[1].cpu().numpy()
predict_all = np.append(predict_all, pred)
result = pd.DataFrame(predict_final)
result.to_csv('result.csv', index=None, encoding='utf-8')
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
print('finish!!')
def test(config, model, test_iter):
model.load_state_dict(torch.load('model/' + config.run_name + '.ckpt'))
model.eval()
start_time = time.time()
test_acc_intent, test_intent_loss, test_intent_report, test_intent_confusion, test_slot_loss, test_acc_slot, test_slot_confusion = evaluate(
config, model, test_iter, test=True)
msg_intent = 'Test Intent Loss: {0:>5.2}, Test Intent Acc: {1:>6.2%}'
msg_slot = 'Test Slot Loss: {0:>5.2}, Test Slot Acc: {1:>6.2%}'
print(msg_intent.format(test_intent_loss, test_acc_intent))
print(msg_slot.format(test_slot_loss, test_acc_slot))
print("Precision, Recall and F1-score...")
print(test_intent_report)
print("Confusion Matrix...")
print(test_intent_confusion)
print(test_slot_confusion)
time_dif = get_time_dif(start_time)
print("Time usage: ", time_dif)
def cut_paras_method(x, config, dataset, type):
# 准备预测数据集
start_time = time.time()
predict_model = Predict_Cut_Paras(dataset=dataset,
config=config,
type=type)
predict_model.build_dataset(path=dataset + '/data/test_data.csv')
time_dif = get_time_dif(start_time)
print('Time usage:', time_dif)
# 预测并写入文件
model = x.Model(config).to(config.device)
predict_labels, ids = predict_model.predict(model=model)
predict_model.write_csv(ids,
predict_labels,
path=dataset + '/data/result_cut_paras' +
str(type) + '.csv')
def test(config, model, test_iter):
vocab = pickle.load(open(config.vocab_path, 'rb'))
emotion_dict, _ = pickle.load(open(config.emotion_path, 'rb'))
emotion = [emotion_dict[i] for i in range(len(emotion_dict))]
emotion = torch.LongTensor(emotion).to(config.device)
grammar = open(config.stopwords_path, 'r', encoding='UTF-8').readlines()
grammar = [
vocab.get(x.strip()) for x in grammar if x.strip() in vocab.keys()
]
grammar = torch.LongTensor(grammar).to(config.device)
checkpoint = torch.load(config.save_path)
model.load_state_dict(checkpoint['model_state_dict'])
model.eval()
start_time = time.time()
vocab = pickle.load(open(config.vocab_path, 'rb'))
re_vocab = {token_id: token for token, token_id in vocab.items()}
x = []
# real_y = []
pred_y = []
with torch.no_grad():
for i, (Queries, Responses) in enumerate(test_iter):
output = model.response(Queries, Responses[2], emotion, grammar)
x = x + [sentence(q, re_vocab) for q in Queries[0].cpu().tolist()]
# real_y = real_y + [sentence(r, re_vocab) for r in Responses[0].cpu().tolist()]
pred_y = pred_y + [sentence(r, re_vocab) for r in output]
data = []
for i in range(len(x)):
data.append('Pair {}'.format(i + 1))
data.append('Query: {}'.format(x[i]))
# data.append('Original Response: {}'.format(real_y[i]))
data.append('Generated Response: {}'.format(pred_y[i]))
data.append(' ')
data = pd.DataFrame(data)
data.to_csv(os.path.join(config.save_dic, 'results_token.txt'),
sep='\t',
encoding='utf8',
header=False,
index=False)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
def predict(config, model2, model135, query):
# out model predict result for analysis
start_time = time.time()
input = build_query(config, query)
with torch.no_grad():
outputs2 = model2(input)
outputs135 = model135(input)
# scl_loss = scl(outputs, labels)
# loss = torch.tensor([cross_loss, scl_loss]) * model.loss_weight
result2 = torch.max(outputs2.data, 1)[1].cpu().numpy()
result135 = torch.max(outputs135.data, 1)[1].cpu().numpy()
print('bi class:{}'.format(result2))
print('bi class:{}'.format(outputs2.data))
print('multi class {}'.format(result135))
print('multi class {}'.format(torch.max(outputs135.data, 1)[0]))
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
