def caculate_bleu(train_data,sentence): logging.debug('=' * 10 + 'BLEU' + '=' * 10) logging.debug(u'句子:%s' %(sentence)) # 创建一个列表存放每个实例的分数 get_wer_score = lambda x: bleu([x.split('|')], sentence.split('|')) sentence_wer_score_list = train_data['SEGMENT_FULL'].apply(get_wer_score).as_matrix() # print sentence_wer_score_list # 排序 sorted_index = np.argsort(sentence_wer_score_list) sorted_index = sorted_index[-1::-1] most_similary_score = sentence_wer_score_list[sorted_index] indexes = sorted_index[:k].tolist() for index,items in enumerate(most_similary_score[k+1:],start=k+1): if items==most_similary_score[0]: indexes.append(sorted_index[index]) else: break most_similary_score = sentence_wer_score_list[indexes] # print most_similary_score most_similary_sentence = train_data['SEGMENT_FULL'].iloc[indexes].as_matrix() most_similary_label = train_data['LABEL'].iloc[indexes].as_matrix() logging.debug(u'前k个最相似的句子为:%s' % (' , '.join(most_similary_sentence))) logging.debug('分数分别为:%s' % most_similary_score) logging.debug(u'类别分别为:%s' % (' , '.join(most_similary_label))) return most_similary_score.tolist(), most_similary_sentence.tolist(), most_similary_label.tolist() similary_score_list = []
def caculate_bleu(train_data,sentence): logging.debug('=' * 10 + 'BLEU' + '=' * 10) similary_score_list = [] for label,group in train_data.groupby(by=['LABEL']): logging.debug( '-'*20) logging.debug(u'正在计算类别(%s),个数:%d'%(label,len(group))) references = [items.split('|') for items in group['SEGMENT_FULL']] # 每个句子和分数绑定在一起 bleu_score = [('|'.join(items),bleu([items], sentence.split('|'))) for items in references] bleu_score = sorted(bleu_score,key=lambda x: x[1],reverse=True) logging.debug('得分列表(从大到小,越大越相似)为:%s'%([item[1] for item in bleu_score])) logging.debug(u'对应句子(从大到小,越大越相似)为:%s'%(','.join([item[0] for item in bleu_score]))) logging.debug(u'最大bleu相似性(越越相似):%f'%(bleu_score[0][1])) similary_score_list.append((label,bleu_score[0][1])) similary_score_list = sorted(similary_score_list,key=lambda x:x[1],reverse=True) most_similary_score = similary_score_list[0][1] most_similary_label = [items[0] for items in similary_score_list if items[1]==most_similary_score or items[1]>0.6] logging.debug( '-'*20) logging.debug(u'最近似类别:%s,分数(%f)'%('|'.join(most_similary_label),most_similary_score)) return most_similary_label,similary_score_list