def evaluate(self, sess, data_manager, id_to_tag): """ :param sess: session to run the model :param data: list of data :param id_to_tag: index to tag name :return: evaluate result """ results = [] trans = self.trans.eval() for batch in data_manager.iter_batch( ): # batch dimension: 4*BatchSize*MaxLenSentence strings = batch[0] # string dimension: BatchSize*MaxLenSentence tags = batch[-1] # tags dimension: BatchSize*MaxLenSentence lengths, scores = self.run_step( sess, False, batch ) # score就是self.logits的计算结果 BatchSize*MaxLenSentence*num_tags batch_paths = self.decode( scores, lengths, trans) # batch_paths dimension: BatchSize*每句实际长度 # 每个值为每个字符由模型算出的对应标签ID for i in range(len(strings)): # 从batch中取出一句 result = [] string = strings[i][:lengths[i]] # string dimension: 第i句实际长度 gold = iobes_iob([ id_to_tag[int(x)] for x in tags[i][:lengths[i]] ]) # gold dimension: 第i句实际长度 pred = iobes_iob([ id_to_tag[int(x)] for x in batch_paths[i][:lengths[i]] ]) # pred dimension: 第i句实际长度 for char, gold, pred in zip(string, gold, pred): # 循环一次一个句子 result.append(" ".join([char, gold, pred])) results.append(result) return results # dimension: eval/test样本数量*每句实际长度 每个三中第一个为字符,第二个为正确标签,第三个为预测标签
def evaluate(self, sess, data, id_to_tag): """ :param sess: session to run the model :param data: list of data :param id_to_tag: index to tag name :return: evaluate result """ results = [] #loss = [] trans = self.trans.eval() chars, segs, string, tags, sequence_length = seperate_input(data) batch = [chars, segs, tags, sequence_length] lengths, scores, loss = self.run_step(sess, False, batch) for item in data: str_lines = item["string"] seqlen = item["seqlen"] tags = [item["tags"][:seqlen]] #loss.append(batch_loss) lengths = [seqlen] batch_paths = self.decode(scores, lengths, trans) for i in range(len(tags)): result = [] strings = [str_lines][i][:lengths[i]] tags = iobes_iob( [id_to_tag[int(x)] for x in tags[i][:lengths[i]]]) preds = iobes_iob( [id_to_tag[int(x)] for x in batch_paths[i][:lengths[i]]]) for char, gold, pred in zip(strings, tags, preds): result.append(" ".join([char, gold, pred])) results.append(result) return results, np.mean(loss)
def evaluate(self, sess, data_manager, id_to_tag): """ :param sess: session to run the model :param data: list of data :param id_to_tag: index to tag name :return: evaluate result """ results = [] trans = self.trans.eval() #转移矩阵的验证 for batch in data_manager.iter_batch(): strings = batch[0] tags = batch[-1] lengths, scores = self.run_step(sess, False, batch) #将验证集进行前向计算得到字数和输出 batch_paths = self.decode(scores, lengths, trans) #得到这一批句子中每个字的预测的命名实体的id for i in range(len(strings)): #遍历每个句子 result = [] string = strings[i][:lengths[i]] #这句话的前i个字,实际上就是这批句子的每句话 gold = iobes_iob([ id_to_tag[int(x)] for x in tags[i][:lengths[i]] ]) #将真实的tag转化为iobes的形式 pred = iobes_iob([ id_to_tag[int(x)] for x in batch_paths[i][:lengths[i]] ]) #将预测的tag转化为iobes的形式 for char, gold, pred in zip(string, gold, pred): result.append(" ".join([char, gold, pred])) #将句子等打包作为输出 results.append(result) return results
def get_reward(self,sess, batch, lengths, scores, id_to_tag): results = [] strings = batch[0] tags = batch[-1] trans = self.trans.eval() batch_paths = self.decode(scores, lengths, trans) for i in range(len(strings)): result = [] string = strings[i][:lengths[i]] gold = iobes_iob([id_to_tag[int(x)] for x in tags[i][:lengths[i]]]) pred = iobes_iob([id_to_tag[int(x)] for x in batch_paths[i][:lengths[i]]]) for char, gold, pred in zip(string, gold, pred): result.append(" ".join([char, gold, pred])) results.append(result) predicts = [] targets = [] rewards = [] for item in results: predict = [ ner_item.split(" ")[1] for ner_item in item] target = [ner_item.split(" ")[2] for ner_item in item] if predict != target: # print( "wrong", predict, target) rewards.append(0) else: rewards.append(1) predicts.append(predict) targets.append(target) rewards = self.effective_rl.compute_sentence_reward(predicts, targets) # print(rewards) assert(len(strings) == len(predicts))
def evaluate(self, sess, data, id_to_tag): """ :param sess: session to run the model :param data: list of data :param id_to_tag: index to tag name :return: evaluate result """ results = [] trans = self.trans.eval() for item in data: batch = create_input(item) str_lines = item["string"] tags = [item["tags"]] lengths, scores = self.run_step(sess, False, batch) lengths = [lengths] batch_paths = self.decode(scores, lengths, trans) for i in range(len(tags)): result = [] strings = [str_lines][i][:lengths[i]] tags = iobes_iob( [id_to_tag[int(x)] for x in tags[i][:lengths[i]]]) preds = iobes_iob( [id_to_tag[int(x)] for x in batch_paths[i][:lengths[i]]]) for char, gold, pred in zip(strings, tags, preds): result.append(" ".join([char, gold, pred])) results.append(result) return results
def evaluate(self, sess, data_manager, id_to_tag): """ :param sess: session to run the model :param data: list of data :param id_to_tag: index to tag name :return: evaluate result """ results = [] trans = self.trans.eval() for batch in data_manager.iter_batch(): strings = batch[0] tags = batch[-1] lengths, scores = self.run_step(sess, False, batch) batch_paths = self.decode(scores, lengths, trans) for i in range(len(strings)): result = [] string = strings[i][:lengths[i]] gold = iobes_iob( [id_to_tag[int(x)] for x in tags[i][:lengths[i]]]) pred = iobes_iob( [id_to_tag[int(x)] for x in batch_paths[i][:lengths[i]]]) for char, gold, pred in zip(string, gold, pred): result.append(" ".join([char, gold, pred])) results.append(result) return results
def eval_model(id_to_char, id_to_tag, test_manager, device, model_name=None): print("Eval ......") if not model_name: model_name = args.log_name old_weights = np.random.rand(len(id_to_char), args.word_embed_dim) pre_word_embed = load_word2vec("100.utf8", id_to_char, args.word_embed_dim, old_weights) e_model = Model(args, id_to_tag, device, pre_word_embed).to(device) e_model.load_state_dict(torch.load("./models/" + model_name + ".pkl")) print("model loaded ...") e_model.eval() all_results = [] for batch in test_manager.iter_batch(): strs, lens, chars, segs, subtypes, tags, adj, dep = batch chars = torch.LongTensor(chars).to(device) _lens = torch.LongTensor(lens).to(device) subtypes = torch.LongTensor(subtypes).to(device) tags = torch.LongTensor(tags).to(device) adj = torch.FloatTensor(adj).to(device) dep = torch.LongTensor(dep).to(device) logits, _ = e_model(chars, _lens, subtypes, adj, dep) """ Evaluate """ # Decode batch_paths = [] for index in range(len(logits)): length = lens[index] score = logits[index][:length] # [seq, dim] probs = F.softmax(score, dim=-1) # [seq, dim] path = torch.argmax(probs, dim=-1) # [seq] batch_paths.append(path) for i in range(len(strs)): result = [] string = strs[i][:lens[i]] gold = iobes_iob([id_to_tag[int(x)] for x in tags[i][:lens[i]]]) pred = iobes_iob( [id_to_tag[int(x)] for x in batch_paths[i][:lens[i]]]) for char, gold, pred in zip(string, gold, pred): result.append(" ".join([char, gold, pred])) all_results.append(result) all_eval_lines = test_ner(all_results, args.result_path, args.log_name) res_info = all_eval_lines[1].strip() f1 = float(res_info.split()[-1]) print("eval: f1: {}".format(f1)) return f1, res_info
def run_step(self, sess, is_train, batch, need_reward=False, id_to_tag=dict()): """ :param sess: session to run the batch :param is_train: a flag indicate if it is a train batch :param batch: a dict containing batch data :return: batch result, loss of the batch or logits """ feed_dict = self.create_feed_dict(is_train, batch) if is_train: global_step, loss, _, lengths, action_prob = sess.run( [self.global_step, self.loss, self.train_op, self.lengths, self.action_prob], feed_dict) rewards = None if need_reward: strings = batch[0] tags = batch[-1] target_tags = [] for i in range(len(strings)): gold = iobes_iob([id_to_tag[int(x)] for x in tags[i][:lengths[i]]]) target_tags.append(gold) rewards = self.effective_rl.compute_tag_reward(target_tags) # print(rewards.shape, "!!!!!!!!!!!!! the tag rewards", rewards.shape[1], rewards.shape[1]>1) # rewards = np.reshape(rewards, [-1]) return global_step, loss, rewards, action_prob else: lengths, logits = sess.run([self.lengths, self.logits], feed_dict) return lengths, logits
def evaluate(self, sess, data_num, char_to_id, tag_to_id, id_to_tag, data_type): """ :param sess: session to run the model :param data: list of data :param id_to_tag: index to tag name :return: evaluate result """ results = [] trans = self.trans.eval() total_loss = [] for j in range(data_num): if data_type == "test": tag_path = 'NER_test/test_label' + str(j + 1) + '.npy' feature_path = 'NER_test/elmo_result' + str(j + 1) + '.npy' str_path = 'NER_test/test_sentence' + str(j + 1) + '.npy' if data_type == "dev": tag_path = 'NER_dev/dev_label' + str(j + 1) + '.npy' feature_path = 'NER_dev/elmo_result' + str(j + 1) + '.npy' str_path = 'NER_dev/dev_sentence' + str(j + 1) + '.npy' tag_load = np.load(tag_path) feature_load = np.load(feature_path) str_load = np.load(str_path) data_tags = [] for sen in tag_load: tag = [] for item in sen: tag.append(tag_to_id[item]) data_tags.append(tag) batch = convert_feature_tag(str_load, char_to_id, feature_load, data_tags) strings = batch[0] tags = batch[5] lengths, scores, testloss = self.run_step(sess, False, batch) total_loss.append(testloss) batch_paths = self.decode(scores, lengths, trans) for i in range(len(strings)): result = [] string = strings[i][:lengths[i]] gold = iobes_iob( [id_to_tag[int(x)] for x in tags[i][:lengths[i]]]) pred = iobes_iob( [id_to_tag[int(x)] for x in batch_paths[i][:lengths[i]]]) for char, gold, pred in zip(string, gold, pred): result.append(" ".join([char, gold, pred])) results.append(result) return results, total_loss
def evaluate(self, sess, data_manager, id_to_tag): results = [] for batch in data_manager.iter_batch(): strings = batch[0] tags = batch[-1] lengths, scores = self.run_step(sess, False, batch) batch_paths = self.decode(scores, lengths) for i in range(len(strings)): result = [] string = strings[i][:lengths[i]] gold = iobes_iob([id_to_tag[int(x)] for x in tags[i][:lengths[i]]]) pred = iobes_iob([id_to_tag[int(x)] for x in batch_paths[i][:lengths[i]]]) for char, gold, pred in zip(string, gold, pred): result.append(" ".join([char, gold, pred])) results.append(result) return results
def dev_epoch(epoch, model, dev_manager, id_to_tag, device): # dev model.eval() all_results = [] for batch in dev_manager.iter_batch(): strs, lens, chars, segs, subtypes, tags, adj, dep = batch chars = torch.LongTensor(chars).to(device) _lens = torch.LongTensor(lens).to(device) subtypes = torch.LongTensor(subtypes).to(device) tags = torch.LongTensor(tags).to(device) adj = torch.FloatTensor(adj).to(device) dep = torch.LongTensor(dep).to(device) logits, _ = model(chars, _lens, subtypes, adj, dep) # [batch, seq, dim] """ Evaluate """ # Decode batch_paths = [] for index in range(len(logits)): length = lens[index] score = logits[index][:length] # [seq, dim] probs = F.softmax(score, dim=-1) # [seq, dim] path = torch.argmax(probs, dim=-1) # [seq] batch_paths.append(path) for i in range(len(strs)): result = [] string = strs[i][:lens[i]] gold = iobes_iob([id_to_tag[int(x)] for x in tags[i][:lens[i]]]) pred = iobes_iob( [id_to_tag[int(x)] for x in batch_paths[i][:lens[i]]]) for char, gold, pred in zip(string, gold, pred): result.append(" ".join([char, gold, pred])) all_results.append(result) all_eval_lines = test_ner(all_results, args.result_path, args.log_name) log_handler.info("epoch: {}, info: {}".format(epoch + 1, all_eval_lines[1].strip())) f1 = float(all_eval_lines[1].strip().split()[-1]) return f1, model
def evaluate(self, sess, dataset_name, id_to_tag, id_to_char): sess.run(self.make_dataset_init(dataset_name)) results = [] trans = self.trans.eval() try: while True: inputs, tags, lengths, scores = self.run_step(sess, False) batch_paths = self.decode(scores, lengths, trans) for i in range(len(inputs)): inp = inputs[i][:lengths[i]] string = [id_to_char[ip] for ip in inp] gold = iobes_iob( [id_to_tag[int(x)] for x in tags[i][:lengths[i]]]) pred = iobes_iob([ id_to_tag[int(x)] for x in batch_paths[i][:lengths[i]] ]) results.append([string, gold, pred]) except tf.errors.OutOfRangeError: pass return results
def evaluate(self, sess, data_manager, id_to_tag): """ :param sess: session to run the model :param data: list of data :param id_to_tag: index to tag name :return: evaluate result """ results = [] trans = self.trans.eval() for batch in data_manager.iter_batch(): strings = batch[0] tags = batch[-1] lengths, scores = self.run_step(sess, False, batch) batch_paths = self.decode(scores, lengths, trans) for i in range(len(strings)): result = [] string = strings[i][:lengths[i]] gold = iobes_iob([id_to_tag[int(x)] for x in tags[i][:lengths[i]]]) pred = iobes_iob([id_to_tag[int(x)] for x in batch_paths[i][:lengths[i]]]) for char, gold, pred in zip(string, gold, pred): result.append(" ".join([char, gold, pred])) results.append(result) return results
def evaluate(self, sess, data_manager, id_to_tag, precision_loc, precision_per, precision_org): """ :param sess: session to run the model :param data: list of data :param id_to_tag: index to tag name :return: evaluate result """ results = [] trans = self.trans.eval() predds = [] for i in range(len(precision_loc)): pred1 = precision_loc[i] pred2 = precision_per[i] pred3 = precision_org[i] preds = pred1 for j in range(len(pred1)): value1 = pred1[j] value2 = pred2[j] value3 = pred3[j] if value1 == value2 == value3: value = value1 elif value1 == value2 and value2 != value3: value = value1 elif value1 == value3 and value2 != value3: value = value1 elif value2 == value3 and value1 != value3: value = value2 else: value = value1 preds[j] = value predds.append(preds) for batch in data_manager.iter_batch(): strings = batch[0] tags = batch[-1] lengths, scores = self.run_step(sess, False, batch) for i in range(len(strings)): result = [] string = strings[i][:lengths[i]] gold = iobes_iob( [id_to_tag[int(x)] for x in tags[i][:lengths[i]]]) pred = predds[i] for char, gold, pred in zip(string, gold, pred): result.append(" ".join([char, gold, pred])) results.append(result) return results
def run_rl(self, sess, is_train, batch, id_to_tag): feed_dict = self.create_feed_dict(is_train, batch) global_step, loss, lengths, action_prob = sess.run( [self.global_step, self.loss, self.lengths, self.action_prob], feed_dict) strings = batch[0] tags = batch[-1] target_tags = [] for i in range(len(strings)): gold = iobes_iob([id_to_tag[int(x)] for x in tags[i][:lengths[i]]]) target_tags.append(gold) rewards = self.effective_rl.compute_tag_reward(target_tags) # print(rewards.shape, "!!!!!!!!!!!!! the tag rewards", rewards.shape[1], rewards.shape[1]>1) #rewards = np.reshape(rewards, [-1]) feed_dict[self.rewards] = rewards global_step, rl_loss, _ = sess.run([self.global_step, self.rl_loss, self.train_rl_op], feed_dict) return global_step, loss, rl_loss
def precision(self, sess, data_manager, id_to_tag): """ :param sess: session to run the model :param data: list of data :param id_to_tag: index to tag name :return: evaluate result """ preds = [] trans = self.trans.eval() for batch in data_manager.iter_batch(): strings = batch[0] lengths, scores = self.run_step(sess, False, batch) batch_paths = self.decode(scores, lengths, trans) for i in range(len(strings)): pred = [] string = strings[i][:lengths[i]] pred = iobes_iob( [id_to_tag[int(x)] for x in batch_paths[i][:lengths[i]]]) preds.append(pred) return preds
def predict(self, sess, batch, label_vocab): """ Args: sess: session to run the model batch: [strings, char_ids, seg_ids, _] label_vocab: index to tag name Return: result: list of labels """ results = [] trans = self.trans.eval(sess) strings = batch[0] lengths, scores = self.run_step(sess, False, batch) batch_paths = self.decode(scores, lengths, trans) for i in range(len(strings)): result = [] string = strings[i][:lengths[i]] pred = iobes_iob( [label_vocab[int(x)] for x in batch_paths[i][:lengths[i]]]) pred = check_iob(pred) results.append(pred) return results