def summarize(self, text, num=6):
# 切句
if type(text) == str:
sentences = cut_sentence(text)
elif type(text) == list:
sentences = text
else:
raise RuntimeError("text type must be list or str")
# str of sentence >>> index
corpus = _build_corpus(sentences)
# pagerank and so on
most_important_docs = summarize_corpus(corpus)
count = 0
sentences_score = {}
for cor in corpus:
tuple_cor = tuple(cor)
sentences_score[sentences[count]] = most_important_docs[tuple_cor]
count += 1
# 最小句子数
num_min = min(num, int(len(sentences) * 0.6))
score_sen = [(rc[1], rc[0]) for rc in sorted(
sentences_score.items(), key=lambda d: d[1], reverse=True)
][0:num_min]
return score_sen
def summarize(self, text, type='mix', num=3):
"""
lead-s
:param sentences: list
:param type: str, you can choose 'begin', 'end' or 'mix'
:return: list
"""
sentences = cut_sentence(text)
if len(sentences) < num:
return sentences
# 最小句子数
num_min = min(num, len(sentences))
if type == 'begin':
summers = sentences[0:num]
elif type == 'end':
summers = sentences[-num:]
else:
summers = [sentences[0]] + [sentences[-1]] + sentences[1:num - 1]
summers_s = {}
for i in range(len(summers)): # 得分计算
if len(summers) - i == 1:
summers_s[summers[i]] = (num - 0.75) / (num + 1)
else:
summers_s[summers[i]] = (num - i - 0.5) / (num + 1)
score_sen = [(rc[1], rc[0]) for rc in sorted(
summers_s.items(), key=lambda d: d[1], reverse=True)][0:num_min]
return score_sen
def text_summarize(
doc,
num=None,
multi_process=False,
fs=[text_pronouns, text_teaser, mmr, text_rank, lead3, lda, lsi, nmf]):
"""
抽取式文本摘要, 汇总, 使用几个方法
:param doc: str or list, 用户输入
:param num: int, 返回的句子个数
:param multi_process: bool, 是否使用多进程
:return: res_score: list, sentences of doc with score
"""
if type(doc) == list:
doc = "。".join(doc)
elif not doc or (type(doc) != str):
raise RuntimeError(" type of doc must be 'list' or 'str' ")
if not num:
from nlg_yongzhuo.data_preprocess.text_preprocess import cut_sentence
num = len(cut_sentence(doc))
# 是否使用多进程, 注意: 当cpu数量不足或性能较差时, 多进程不一定比串行快
if multi_process:
res = summary_multi_preprocess(doc, num, fs)
else:
res = summary_serial(doc, num, fs)
# 后处理
res_score = summary_post_preprocess(res)
return res_score
def summarize(self, text, num=8, alpha=0.6):
"""
:param text: str
:param num: int
:return: list
"""
# 切句
if type(text) == str:
self.sentences = cut_sentence(text)
elif type(text) == list:
self.sentences = text
else:
raise RuntimeError("text type must be list or str")
# 切词
sentences_cut = [[
word for word in jieba_cut(extract_chinese(sentence))
if word.strip()
] for sentence in self.sentences]
# 去除停用词等
self.sentences_cut = [
list(filter(lambda x: x not in self.stop_words, sc))
for sc in sentences_cut
]
self.sentences_cut = [" ".join(sc) for sc in self.sentences_cut]
# # 计算每个句子的词语个数
# sen_word_len = [len(sc)+1 for sc in sentences_cut]
# 计算每个句子的tfidf
sen_tfidf = tfidf_fit(self.sentences_cut)
# 矩阵中两两句子相似度
SimMatrix = (sen_tfidf *
sen_tfidf.T).A # 例如: SimMatrix[1, 3] # "第2篇与第4篇的相似度"
# 输入文本句子长度
len_sen = len(self.sentences)
# 句子标号
sen_idx = [i for i in range(len_sen)]
summary_set = []
mmr = {}
for i in range(len_sen):
if not self.sentences[i] in summary_set:
sen_idx_pop = copy.deepcopy(sen_idx)
sen_idx_pop.pop(i)
# 两两句子相似度
sim_i_j = [SimMatrix[i, j] for j in sen_idx_pop]
score_tfidf = sen_tfidf[i].toarray()[0].sum(
) # / sen_word_len[i], 如果除以词语个数就不准确
mmr[self.sentences[i]] = alpha * score_tfidf - (
1 - alpha) * max(sim_i_j)
summary_set.append(self.sentences[i])
score_sen = [
(rc[1], rc[0])
for rc in sorted(mmr.items(), key=lambda d: d[1], reverse=True)
]
if len(mmr) > num:
score_sen = score_sen[0:num]
return score_sen
def summarize(self, text, num=6):
"""
根据词语意义确定中心句
:param text: str
:param num: int
:return: list
"""
# 切句
if type(text) == str:
self.sentences = cut_sentence(text)
elif type(text) == list:
self.sentences = text
else:
raise RuntimeError("text type must be list or str")
# 切词
sentences_cut = [[word for word in jieba_cut(extract_chinese(sentence))
if word.strip()] for sentence in self.sentences]
# 去除停用词等
self.sentences_cut = [list(filter(lambda x: x not in self.stop_words, sc)) for sc in sentences_cut]
# 词频统计
self.words = []
for sen in self.sentences_cut:
self.words = self.words + sen
self.word_count = dict(Counter(self.words))
self.word_count_rank = sorted(self.word_count.items(), key=lambda f: f[1], reverse=True)
# 最小句子数
num_min = min(num, len(self.sentences))
# 词语排序, 按照词频
self.word_rank = [wcr[0] for wcr in self.word_count_rank][0:num_min]
res_sentence = []
# 抽取句子, 顺序, 如果词频高的词语在句子里, 则抽取
for word in self.word_rank:
for i in range(0, len(self.sentences)):
# 当返回关键句子到达一定量, 则结束返回
if len(res_sentence) < num_min:
added = False
for sent in res_sentence:
if sent == self.sentences[i]: added = True
if (added == False and word in self.sentences[i]):
res_sentence.append(self.sentences[i])
break
# 只是计算各得分,没什么用
res_sentence = [(1-1/(len(self.sentences)+1), rs) for rs in res_sentence]
return res_sentence
def textrank_tfidf(sentences, topk=6):
"""
使用tf-idf作为相似度, networkx.pagerank获取中心句子作为摘要
:param sentences: str, docs of text
:param topk:int
:return:list
"""
# 切句子
sentences = list(cut_sentence(sentences))
# tf-idf相似度
matrix_norm = tdidf_sim(sentences)
# 构建相似度矩阵
tfidf_sim = nx.from_scipy_sparse_matrix(matrix_norm * matrix_norm.T)
# nx.pagerank
sens_scores = nx.pagerank(tfidf_sim)
# 得分排序
sen_rank = sorted(sens_scores.items(), key=lambda x: x[1], reverse=True)
# 保留topk个, 防止越界
topk = min(len(sentences), topk)
# 返回原句子和得分
return [(sr[1], sentences[sr[0]]) for sr in sen_rank][0:topk]
def summarize(self, text, num=8, topic_min=6, judge_topic=None):
"""
:param text: str
:param num: int
:return: list
"""
# 切句
if type(text) == str:
self.sentences = cut_sentence(text)
elif type(text) == list:
self.sentences = text
else:
raise RuntimeError("text type must be list or str")
len_sentences_cut = len(self.sentences)
# 切词
sentences_cut = [[
word for word in jieba_cut(extract_chinese(sentence))
if word.strip()
] for sentence in self.sentences]
# 去除停用词等
self.sentences_cut = [
list(filter(lambda x: x not in self.stop_words, sc))
for sc in sentences_cut
]
self.sentences_cut = [" ".join(sc) for sc in self.sentences_cut]
# 计算每个句子的tf
vector_c = CountVectorizer(ngram_range=(1, 2),
stop_words=self.stop_words)
tf_ngram = vector_c.fit_transform(self.sentences_cut)
# 主题数, 经验判断
topic_num = min(topic_min, int(len(sentences_cut) / 2)) # 设定最小主题数为3
lda = LatentDirichletAllocation(n_components=topic_num,
max_iter=32,
learning_method='online',
learning_offset=50.,
random_state=2019)
res_lda_u = lda.fit_transform(tf_ngram.T)
res_lda_v = lda.components_
if judge_topic:
### 方案一, 获取最大那个主题的k个句子
##################################################################################
topic_t_score = np.sum(res_lda_v, axis=-1)
# 对每列(一个句子topic_num个主题),得分进行排序,0为最大
res_nmf_h_soft = res_lda_v.argsort(axis=0)[-topic_num:][::-1]
# 统计为最大每个主题的句子个数
exist = (res_nmf_h_soft <= 0) * 1.0
factor = np.ones(res_nmf_h_soft.shape[1])
topic_t_count = np.dot(exist, factor)
# 标准化
topic_t_count /= np.sum(topic_t_count, axis=-1)
topic_t_score /= np.sum(topic_t_score, axis=-1)
# 主题最大个数占比, 与主题总得分占比选择最大的主题
topic_t_tc = topic_t_count + topic_t_score
topic_t_tc_argmax = np.argmax(topic_t_tc)
# 最后得分选择该最大主题的
res_nmf_h_soft_argmax = res_lda_v[topic_t_tc_argmax].tolist()
res_combine = {}
for l in range(len_sentences_cut):
res_combine[self.sentences[l]] = res_nmf_h_soft_argmax[l]
score_sen = [(rc[1], rc[0]) for rc in sorted(
res_combine.items(), key=lambda d: d[1], reverse=True)]
#####################################################################################
else:
### 方案二, 获取最大主题概率的句子, 不分主题
res_combine = {}
for i in range(len_sentences_cut):
res_row_i = res_lda_v[:, i]
res_row_i_argmax = np.argmax(res_row_i)
res_combine[self.sentences[i]] = res_row_i[res_row_i_argmax]
score_sen = [(rc[1], rc[0]) for rc in sorted(
res_combine.items(), key=lambda d: d[1], reverse=True)]
num_min = min(num, int(len_sentences_cut * 0.6))
return score_sen[0:num_min]
def summarize(self, text, num=6, title=None):
"""
文本句子排序
:param docs: list
:return: list
"""
# 切句
if type(text) == str:
self.sentences = cut_sentence(text)
elif type(text) == list:
self.sentences = text
else:
raise RuntimeError("text type must be list or str")
self.title = title
if self.title:
self.title = jieba_cut(title)
# 切词,含词性标注
self.sentences_tag_cut = [
jieba_tag_cut(extract_chinese(sentence))
for sentence in self.sentences
]
# 词语,不含词性标注
sentences_cut = [[jc for jc in jtc.keys()]
for jtc in self.sentences_tag_cut]
# 去除停用词等
self.sentences_cut = [
list(filter(lambda x: x not in self.stop_words, sc))
for sc in sentences_cut
]
# 词频统计
self.words = []
for sen in self.sentences_cut:
self.words = self.words + sen
self.word_count = dict(Counter(self.words))
# 按频次计算词语的得分, 得到self.word_freq=[{'word':, 'freq':, 'score':}]
self.word_freqs = {}
self.len_words = len(self.words)
for k, v in self.word_count.items():
self.word_freqs[k] = v * 0.5 / self.len_words
# uni_bi_tri_gram特征
gram_uni, gram_bi, gram_tri = gram_uni_bi_tri("".join(self.sentences))
ngrams = gram_uni + gram_bi + gram_tri
self.ngrams_count = dict(Counter(ngrams))
# 句子位置打分
scores_posi = self.score_position()
# 句子长度打分
scores_length = self.score_length()
# 句子词性打分, 名词(1.2)-代词(0.8)-动词(1.0)
scores_tag = self.score_tag()
res_rank = {}
self.res_score = []
for i in range(len(sentences_cut)):
sen_cut = self.sentences_cut[i] # 句子中的词语
# ngram得分
gram_uni_, gram_bi_, gram_tri_ = gram_uni_bi_tri(self.sentences[i])
n_gram_s = gram_uni_ + gram_bi_ + gram_tri_
score_ngram = sum([
self.ngrams_count[ngs] if ngs in self.ngrams_count else 0
for ngs in n_gram_s
]) / (len(n_gram_s) + 1)
# 句子中词语的平均长度
score_word_length_avg = sum([len(sc) for sc in sen_cut
]) / (len(sen_cut) + 1)
score_posi = scores_posi[i]
score_length = scores_length[i]
score_tag = scores_tag[i]
if self.title: # 有标题的文本打分合并
score_title = self.score_title(sen_cut)
score_total = (score_title * 0.5 + score_ngram * 2.0 +
score_word_length_avg * 0.5 + score_length * 0.5
+ score_posi * 1.0 + score_tag * 0.6) / 6.0
# 可查阅各部分得分统计
self.res_score.append([
"score_title", "score_ngram", "score_word_length_avg",
"score_length", "score_posi", "score_tag"
])
self.res_score.append([
score_title, score_ngram, score_word_length_avg,
score_length, score_posi, score_tag, self.sentences[i]
])
else: # 无标题的文本打分合并
score_total = (score_ngram * 2.0 + score_word_length_avg * 0.5
+ score_length * 0.5 + score_posi * 1.0 +
score_tag * 0.6) / 5.0
# 可查阅各部分得分统计
self.res_score.append([
"score_ngram", "score_word_length_avg", "score_length",
"score_posi", "score_tag"
])
self.res_score.append([
score_ngram, score_word_length_avg, score_length,
score_posi, score_tag, self.sentences[i]
])
res_rank[self.sentences[i].strip()] = score_total
# 最小句子数
num_min = min(num, int(len(self.word_count) * 0.6))
res_rank_sort = sorted(res_rank.items(),
key=lambda rr: rr[1],
reverse=True)
res_rank_sort_reverse = [(rrs[1], rrs[0])
for rrs in res_rank_sort][0:num_min]
return res_rank_sort_reverse
def summarize(self, text, num=8, topic_min=3, judge_topic="all"):
"""
:param text: text or list, input docs
:param num: int, number or amount of return
:param topic_min: int, topic number
:param judge_topic: str, calculate ways of topic
:return:
"""
# 切句
if type(text) == str:
self.sentences = cut_sentence(text)
elif type(text) == list:
self.sentences = text
else:
raise RuntimeError("text type must be list or str")
# 切词
sentences_cut = [[
word for word in jieba_cut(extract_chinese(sentence))
if word.strip()
] for sentence in self.sentences]
len_sentences_cut = len(sentences_cut)
# 去除停用词等
self.sentences_cut = [
list(filter(lambda x: x not in self.stop_words, sc))
for sc in sentences_cut
]
self.sentences_cut = [" ".join(sc) for sc in self.sentences_cut]
# 计算每个句子的tfidf
sen_tfidf = tfidf_fit(self.sentences_cut)
# 主题数, 经验判断
topic_num = min(topic_min, int(len(sentences_cut) / 2)) # 设定最小主题数为3
nmf_tfidf = NMF(n_components=topic_num, max_iter=320)
res_nmf_w = nmf_tfidf.fit_transform(sen_tfidf.T) # 基矩阵 or 权重矩阵
res_nmf_h = nmf_tfidf.components_ # 系数矩阵 or 降维矩阵
if judge_topic:
### 方案一, 获取最大那个主题的k个句子
##################################################################################
topic_t_score = np.sum(res_nmf_h, axis=-1)
# 对每列(一个句子topic_num个主题),得分进行排序,0为最大
res_nmf_h_soft = res_nmf_h.argsort(axis=0)[-topic_num:][::-1]
# 统计为最大每个主题的句子个数
exist = (res_nmf_h_soft <= 0) * 1.0
factor = np.ones(res_nmf_h_soft.shape[1])
topic_t_count = np.dot(exist, factor)
# 标准化
topic_t_count /= np.sum(topic_t_count, axis=-1)
topic_t_score /= np.sum(topic_t_score, axis=-1)
# 主题最大个数占比, 与主题总得分占比选择最大的主题
topic_t_tc = topic_t_count + topic_t_score
topic_t_tc_argmax = np.argmax(topic_t_tc)
# 最后得分选择该最大主题的
res_nmf_h_soft_argmax = res_nmf_h[topic_t_tc_argmax].tolist()
res_combine = {}
for l in range(len_sentences_cut):
res_combine[self.sentences[l]] = res_nmf_h_soft_argmax[l]
score_sen = [(rc[1], rc[0]) for rc in sorted(
res_combine.items(), key=lambda d: d[1], reverse=True)]
#####################################################################################
else:
### 方案二, 获取最大主题概率的句子, 不分主题
res_combine = {}
for i in range(len_sentences_cut):
res_row_i = res_nmf_h[:, i]
res_row_i_argmax = np.argmax(res_row_i)
res_combine[self.sentences[i]] = res_row_i[res_row_i_argmax]
score_sen = [(rc[1], rc[0]) for rc in sorted(
res_combine.items(), key=lambda d: d[1], reverse=True)]
num_min = min(num, len(self.sentences))
return score_sen[0:num_min]
def summarize(self, text, num=6, title=None):
# 切句
if type(text) == str:
self.sentences = cut_sentence(text)
elif type(text) == list:
self.sentences = text
else:
raise RuntimeError("text type must be list or str")
self.title = title
if self.title:
self.title = jieba_cut(title)
# 切词
sentences_cut = [[
word for word in jieba_cut(extract_chinese(sentence))
if word.strip()
] for sentence in self.sentences]
# 去除停用词等
self.sentences_cut = [
list(filter(lambda x: x not in self.stop_words, sc))
for sc in sentences_cut
]
# 词频统计
self.words = []
for sen in self.sentences_cut:
self.words = self.words + sen
self.word_count = dict(Counter(self.words))
# word_count_rank = sorted(word_count.items(), key=lambda f:f[1], reverse=True)
# self.word_freqs = [{'word':wcr[0], 'freq':wcr[1]} for wcr in word_count_rank]
# 按频次计算词语的得分, 得到self.word_freq=[{'word':, 'freq':, 'score':}]
self.word_freqs = {}
self.len_words = len(self.words)
for k, v in self.word_count.items():
self.word_freqs[k] = v * 0.5 / self.len_words
# 句子位置打分
scores_posi = self.score_position()
res_rank = {}
self.res_score = []
for i in range(len(sentences_cut)):
sen = self.sentences[i] # 句子
sen_cut = self.sentences_cut[i] # 句子中的词语
score_sbs = self.score_sbs(sen_cut) # 句子中的词语打分1
score_dbs = self.score_dbs(sen_cut) # 句子中的词语打分2
score_word = (score_sbs + score_dbs) * 10.0 / 2.0 # 句子中的词语打分mix
score_length = self.score_length(sen) # 句子文本长度打分
score_posi = scores_posi[i]
if self.title: # 有标题的文本打分合并
score_title = self.score_title(sen_cut)
score_total = (score_title * 0.5 + score_word * 2.0 +
score_length * 0.5 + score_posi * 1.0) / 4.0
# 可查阅各部分得分统计
self.res_score.append([
"score_total", "score_sbs", "score_dbs", "score_word",
"score_length", "score_posi", "score_title", "sentences"
])
self.res_score.append([
score_total, score_sbs, score_dbs, score_word,
score_length, score_posi, score_title, self.sentences[i]
])
else: # 无标题的文本打分合并
score_total = (score_word * 2.0 + score_length * 0.5 +
score_posi * 1.0) / 3.5
self.res_score.append([
"score_total", "score_sbs", "score_dbs", "score_word",
"score_length", "score_posi", "sentences"
])
self.res_score.append([
score_total, score_sbs, score_dbs, score_word,
score_length, score_posi, self.sentences[i].strip()
])
res_rank[self.sentences[i].strip()] = score_total
# 最小句子数
num_min = min(num, int(len(self.word_count) * 0.6))
score_sen = [(rc[1], rc[0]) for rc in sorted(
res_rank.items(), key=lambda d: d[1], reverse=True)][0:num_min]
return score_sen