def cal_confidence_score(sample, model_name): ''' 给未标注数据打标签,并且计算得分,返回结果 :param model: 模型 :param sample: 对象 :return: 预测的功能点名称 置信度 ''' model = Tagger() model.open(model_name) # unlabeled sample features feature_sequence = build_model_features(sample, 17, False) # words # words = sample.sen_words chars = list(sample.sentence) model.set(feature_sequence) predicted_labels = model.tag() # get predicted_fps fp_list = [] fp = '' for index, label in enumerate(predicted_labels): if label == 'B' or label == 'I' or label == 'E': fp += chars[index] if label == 'N' and len(fp) > 0: fp_list.append(fp) fp = '' # calculate the probability of tagging crf_confidence = model.probability(predicted_labels) lan_confidence = 0 filtered_fp_list = [] for fp_name in fp_list: filtered_fp_list.append(fp_name) if len(filtered_fp_list) == 0: predicted_fps = 'null' else: predicted_fps = ' '.join(filtered_fp_list) # print(str(sample.story_id) +' '+ sample.sentence +' '+ fp +' '+ str(confidence)) # 为防止多进程乱序执行导致结果跟sample不对应,因此同时返回sample信息 return sample.story_id, sample.sentence, predicted_fps, crf_confidence
def crf_predict( tagger: pycrfsuite.Tagger, gp_data: list, mode: str = 'raw', exclude_labels: list = ['NOL', 'NAT', 'NEE'] ) -> Union[list, Tuple[list, pd.DataFrame]]: """Return predictions for the test data, grouped by file. 3 modes for return: * Return raw predictions (raw) * Return predictions with only valid tags (exclude_ool) * Return predictions (valid tags) and probabilities for each class (rt_proba) Predictions are returned unflattened https://python-crfsuite.readthedocs.io/en/latest/pycrfsuite.html """ if mode not in ['raw', 'exclude_ool', 'rt_proba']: raise ValueError( f"mode must be one of raw|exclude_ool|rt_proba; currently {mode}") if mode == 'raw': return [tagger.tag(xseq) for xseq in gp_data] labels = tagger.labels() res = [] y_pred = [] for fi, xseq in enumerate(gp_data): tagger.set(xseq) file_proba = pd.DataFrame({ label: [tagger.marginal(label, i) for i in range(len(xseq))] for label in labels }) y_pred.append(file_proba[[ col for col in file_proba.columns if col not in exclude_labels ]].idxmax(axis=1).tolist()) file_proba['file_id'] = fi res.append(file_proba) if mode == 'rt_proba': return y_pred, pd.concat(res, axis=0) return y_pred # else
def evaluate_model_by_story(model_name, test_samples): model = Tagger() model.open(model_name) story_fps = dict() for sample in test_samples: model.set(build_model_features(sample, 17, True)) predicted_labels = model.tag() chars = list(sample.sentence) predicted_fps = [] fp = '' for index, word in enumerate(predicted_labels): if word == 'E' or word == 'S': fp += chars[index] predicted_fps.append(fp) fp = '' if word == 'B' or word == 'I': fp += chars[index] actual_fps = [fp for fp in sample.fps if fp != '' and fp != 'null' and fp in sample.sentence] filtered_predicted_fps = predicted_fps # for predicted_fp in predicted_fps: # lan_confidence_temp = lmmodel.score(predicted_fp, bos=True, eos=True) / len(predicted_fp) # if len(re.findall('[a-zA-Z0-9+]+', predicted_fp)) > 0: # lan_confidence_temp += 5 # if lan_confidence_temp > -2.4: # filtered_predicted_fps.append(predicted_fp) if sample.story_id not in story_fps: story_fps[sample.story_id] = [set(actual_fps), set(filtered_predicted_fps)] else: story_fps[sample.story_id][0].update(actual_fps) story_fps[sample.story_id][1].update(filtered_predicted_fps) # print(len(story_fps)) global sim_t sim_threshold = sim_t TP_precision = 0 TP_recall = 0 all_actual_fps = 0 all_predicted_fps = 0 for story_id, (actual_fps, predicted_fps) in story_fps.items(): story_precision = 0.0 story_recall = 0.0 all_actual_fps += len(actual_fps) all_predicted_fps += len(predicted_fps) # for actual_fp in actual_fps: story = samples_dao.read_story_by_story_id(int(story_id)) data = [story_id, story[0] if story is not None else '', story[1] if story is not None else '', story[2] if story is not None else '', story[3] if story is not None else '', story[4] if story is not None else '', actual_fps, predicted_fps] with open('../Archive/date_performance/resultsIterRes_by_story_details.csv', 'a', newline='') as csv_file: csv_writer = csv.writer(csv_file) csv_writer.writerow(data) for predicted_fp in predicted_fps: sim = [] for actual_fp in actual_fps: similarity = 1-distance.nlevenshtein(actual_fp, predicted_fp, method=1) # if actual_fp in predicted_fp: # similarity = 1 sim.append(similarity) # print(sim) if len(sim) == 0: sim = [0] if max(sim) >= sim_threshold: TP_precision += 1 story_precision += 1 for actual_fp in actual_fps: sim = [] for predicted_fp in predicted_fps: similarity = 1-distance.nlevenshtein(actual_fp, predicted_fp, method=1) sim.append(similarity) # print(sim) if len(sim) == 0: sim = [0] if max(sim) >= sim_threshold: TP_recall += 1 story_recall += 1 # 每个故事的详情 story_precision = 0 if len(filtered_predicted_fps) == 0 else story_precision/len(filtered_predicted_fps) story_recall = 0 if len(actual_fps) == 0 else story_recall/len(actual_fps) data = ["STORY " + story_id, story_precision, story_recall] with open('../Archive/date_performance/results/story_details.csv', 'a', newline='') as csv_file: csv_writer = csv.writer(csv_file) csv_writer.writerow(data) with open('../Archive/date_performance/results/story_details.csv', 'a', newline='') as csv_file: csv_writer = csv.writer(csv_file) csv_writer.writerow(["THE END!!!"]) # 整体的详情 precision = TP_precision/all_predicted_fps recall = TP_recall/all_actual_fps f1 = 2 * precision * recall / (precision + recall) print("By Story: Iteration: %s\n\tPrecision: %f\n\tRecall: %f\n\tF1: %f\n\n\n" % (model_name.split('_')[2], precision, recall, f1)) data = ["BY STORY: Iteration " + model_name.split('_')[2], precision, recall, f1] with open('../Archive/date_performance/results/IterRes_by_story.csv', 'a', newline='') as csv_file: csv_writer = csv.writer(csv_file) csv_writer.writerow(data) return precision, recall, f1
def evaluate_model(model_name, test_samples): ''' 最后一次迭代训练模型,并输出测试结果 :param test_samples: :param model_name: :return: ''' model = Tagger() model.open(model_name) accuracy = 0.0 recall = 0.0 f1 = 0.0 # sample_accuracy = 0.0 iteration_test_details = [] for sample in test_samples: model.set(build_model_features(sample, 17, True)) predicted_labels = model.tag() true_labels = sample.char_label predicted_label_index = [] for predicted_label in predicted_labels: if predicted_label == 'N': predicted_label_index.append(0) else: predicted_label_index.append(1) true_label_index = [] for true_label in true_labels: if true_label == 'N': true_label_index.append(0) else: true_label_index.append(1) iteration_test_details = [] chars = list(sample.sentence) # sen_words = sample.sen_words iteration_test_details.append(sample.sentence) predicted_fps = '' actual_fps = '' for index, word in enumerate(predicted_labels): if word != 'N': predicted_fps += chars[index] if len(predicted_fps) == 0: predicted_fps = '-----' for index, word in enumerate(true_labels): if word != 'N': actual_fps += chars[index] iteration_test_details.append(actual_fps) iteration_test_details.append(predicted_fps) with open('../Archive/date_performance/results/Iteration_Test_Details.csv', 'a', newline='') as csv_file: csv_writer = csv.writer(csv_file) csv_writer.writerow(iteration_test_details) # print(sample.sen_words) # print(predicted_labels) # print(true_labels) accuracy += metrics.accuracy_score(true_label_index, predicted_label_index) recall += metrics.recall_score(true_label_index, predicted_label_index, average='binary', pos_label=1) f1 += 2*accuracy*recall/(accuracy+recall) # sample_accuracy += metrics.sequence_accuracy_score(true_labels, predicted_labels) print("Iteration: %s\n\tAccuracy: %f\n\tRecall: %f\n\tF1: %f\n\n\n" % ( model_name.split('_')[2], accuracy / len(test_samples), recall / len(test_samples), f1 / len(test_samples))) data = ["Iteration " + model_name.split('_')[2], accuracy / len(test_samples), recall / len(test_samples), f1 / len(test_samples)] with open('../Archive/date_performance/results/IterRes.csv', 'a', newline='') as csv_file: csv_writer = csv.writer(csv_file) csv_writer.writerow(data) with open('../Archive/date_performance/results/Iteration_Test_Details.csv', 'a', newline='') as csv_file: csv_writer = csv.writer(csv_file) csv_writer.writerow(data) return accuracy / len(test_samples), recall / len(test_samples), f1 / len(test_samples)