def inference_no_start_bert_server():
net.load_state_dict(torch.load(f'{model_path}/model_80.pth'))
test_x, test_y, test_data = read_data_clean('data/example.test')
while True:
index = input('句子0/1/2/.....(都是测试集的句子)')
sentence = test_x[int(index)]
train_x_ = (np.load(f'data/test_npy/{index}.npy'))
train_x_ = torch.Tensor(train_x_).unsqueeze(0)
train_x_ = to_gpu(train_x_).float()
tag_seq = net.inference(train_x_)
prediction_label = [new_dict.get(i) for i in tag_seq[1:-1]]
print([
sentence[i] + ":" + prediction_label[i]
for i in range(len(prediction_label))
])
flag = False
for i in range(len(prediction_label)):
if prediction_label[i] != 'O':
if flag == False:
flag = True
print(sentence[i], ":", prediction_label[i], end=" ")
elif prediction_label[i] == 'O' and flag == True:
flag = False
print(" ")
print("")
def cal_inference():
net.load_state_dict(torch.load(f'{model_path}/model_90.pth'))
from exc_text import read_data_clean
test_x, test_y, test_data = read_data_clean('data/example.test')
conlleval = []
for index in range(len(test_data)):
print(index / len(test_data))
data_line = test_data[index]
string_x = [token[0] for token in data_line]
string_y = [token[1] for token in data_line]
train_x_ = [string_id_x.get('BIN')
] + [string_id_x[token]
for token in string_x] + [string_id_x.get('EOS')]
train_x_ = torch.IntTensor(train_x_).unsqueeze(0)
train_x_ = to_gpu(train_x_).long()
y_predit = net.inference(train_x_)
prediction = torch.max(F.softmax(y_predit, 2), 2)[1]
text = list(string_x)
prediction_label = [
new_dict.get(i) for i in prediction[0].cpu().numpy()[1:-1]
]
for i in range(len(data_line)):
conlleval.append('{} {} {}'.format(string_x[i], string_y[i],
prediction_label[i]))
from cal_f1 import get_result
res = get_result(conlleval)
print(res)
def cal_inference():
net.eval()
# net.load_state_dict(torch.load(f'{model_path}/model_86.pth')) #最强
net.load_state_dict(torch.load(f'{model_path}/model_80.pth')) # 最强
test_x, test_y, test_data = read_data_clean('data/example.test')
test_x_matri = []
for index in range(len(test_data)):
print(index)
test_x_matri.append(np.load(f'data/test_npy/{index}.npy'))
test_data_wait_load = TextLoader(test_data, test_x_matri)
test_loader = DataLoader(test_data_wait_load,
batch_size=1,
shuffle=False,
collate_fn=collate)
conlleval = []
for index, batch in enumerate(test_loader):
print(index / len(test_data))
data_line = test_data[index]
string_x = [token[0] for token in data_line]
string_y = [token[1] for token in data_line]
x, y, input_lengths = parse_batch(batch)
tag_seq = net.inference(x)
prediction_label = [new_dict.get(i) for i in tag_seq[1:-1]]
for i in range(len(data_line)):
conlleval.append('{} {} {}'.format(string_x[i], string_y[i],
prediction_label[i]))
res = get_result(conlleval)
print(res)
net.train()
def create_vector():
from exc_text import read_data_clean, read_data
test_x, test_y, test_data = read_data_clean('data/example.test')
for i in range(len(test_data)):
data = test_x[i]
data = ['BIN'] + data + ['EOS']
target_data = get_bert_vector(data)
np.save(f'data/test_npy/{i}', target_data)
print(i / len(test_data))
for Linear in self.Linears:
outputs = F.relu(Linear(outputs))
outputs = self.last(outputs)
score, tag_seq = self.CRF_layer(outputs)
return tag_seq
net = NERModel().to(device)
optimizer = torch.optim.Adam(net.parameters(), lr=0.005)
criterion = torch.nn.CrossEntropyLoss()
collate = TextCollate()
from exc_text import read_data_clean
test_x, test_y, test_data = read_data_clean('data/example.test')
def val(time):
net.eval()
test_data_wait_load = TextLoader(test_data)
test_loader = DataLoader(test_data_wait_load,
batch_size=1,
shuffle=False,
collate_fn=collate)
acc = 0
conlleval = []
for index, batch in enumerate(test_loader):
data_line = test_data[index]
string_x = [token[0] for token in data_line]
string_y = [token[1] for token in data_line]