def main(cls, argv): query = TermQuery(Term("f", "ipsum")) scorer = QueryScorer(query) formatter = SimpleHTMLFormatter("<span class=\"highlight\">", "</span>") highlighter = Highlighter(formatter, scorer) fragmenter = SimpleFragmenter(50) highlighter.setTextFragmenter(fragmenter) analyzer = StandardAnalyzer(Version.LUCENE_CURRENT) tokenStream = analyzer.tokenStream("f", StringReader(cls.text)) result = highlighter.getBestFragments(tokenStream, cls.text, 5, "...") stdout.write("<html>") stdout.write("<style>\n") stdout.write(".highlight {\n") stdout.write(" background: yellow\n") stdout.write("}\n") stdout.write("</style>") stdout.write("<body>") stdout.write(result) stdout.write("</body></html>\n") stdout.flush()
class MatrixMapper(object): def __init__(self, STOP_WORDS=StopAnalyzer.ENGLISH_STOP_WORDS): self.pe = pextractor.PhraseExtractor() self.STOP_WORDS = STOP_WORDS self.analyzer = StandardAnalyzer(STOP_WORDS) self.entries = [] def get_cs_by_lucene_doc(self, docs, context): doc_size = len(docs) lucene_ids = [] categories = [] for id in range(doc_size): link = docs[id].get("link") lucene_ids.append(int(docs[id].get("id"))) entry = dao.get_by_link(link, Entry) self.entries.append(entry) # TODO boost title field summary = entry.summary[:200] #if entry.category != '其他': #categories.append(entry.category) stream = self.analyzer.tokenStream("summary", StringReader(summary)) for s in stream: context.tokens.append(s.term()) context.token_types.append(s.type()) stream = self.analyzer.tokenStream("title", StringReader(entry.title)) for s in stream: context.title_field.append(len(context.tokens)) context.tokens.append(s.term()) context.token_types.append(s.type()) context.term_doc_range.append(len(context.tokens)) #print 'tokens:',len(context.tokens) return self.pe.extract(context), lucene_ids, categories def add2matrix(self, tpv, all, term_row, lucene_ids, i,term_doc_freq): for (t, f) in zip(tpv.getTerms(), tpv.getTermFrequencies()): term = [0 for j in range(len(lucene_ids))] new = False if not term_row.has_key(t): term_row[t] = len(term_row) new = True row = term_row[t] if new: term[i] = f#self.tfidf(len(tpv.getTerms()), f, total_terms, dtf) all.append(term) else: all[row][i] = f term_doc_freq[t] = term_doc_freq.get(t,0) + 1 """ 效率很低,弃用 """ def product(self, termmatrix, labelmatrix): row = [] for i in range(len(termmatrix)): col = [] for j in range(len(labelmatrix[0])): #k是a的列,b的行 e = [] for k in range(len(termmatrix[0])): c = termmatrix[i][k] d = labelmatrix[k][j] e.append(c * d) col.append(sum(e)) row.append(col) return row """ 取代label_assign方法 """ def assign(self, docs, labels, lucene_ids): term_row = {} all = [] ireader = IndexReader.open(STORE_DIR) total_terms = 0 term_doc_freq = {} for i in range(len(lucene_ids)): tpv = TermPositionVector.cast_(ireader.getTermFreqVector(lucene_ids[i], 'summary')) self.add2matrix(tpv, all, term_row, lucene_ids, i,term_doc_freq) """ TODO:给属于标题的term加权 """ tpv = TermPositionVector.cast_(ireader.getTermFreqVector(lucene_ids[i], 'title')) self.add2matrix(tpv, all, term_row, lucene_ids, i,term_doc_freq) #for k,v in term_doc_freq.items(): # if v> 3: # print k,v # 对label进行分词 analyzer = CJKAnalyzer() labelmatrix = zeros((len(all), len(labels))) label_term = [] # doc -label:每个doc对应的label all_weight_table = {} #label -doc:每个label对应的doc label_doc = [] label_doc_map = {} for i in range(len(labels)): nonzero_table = [] # 一个label对应和所有doc的权重之积 weight_table = [] stream = analyzer.tokenStream('', StringReader(labels[i].text)) terms = [] c = 0 weight_row = {} nonzero_index = [] is_incomplete = False for token in stream: term = token.term()#token.decode('utf-8')# #print term if term_row.has_key(term): row = term_row[term] terms.append(term) docs_with_current_term = all[row] for j in range(len(docs_with_current_term)): if docs_with_current_term[j] != 0: if c == 0: nonzero_index.append(j) if c == 0 or j in nonzero_index: weight_row[j] = weight_row.get(j, 0) + docs_with_current_term[j] * term_doc_freq[term] * labels[i].label_weight else: # 加1防止权重之积为0 # 针对第一次出现在nonzero_index中而在后面的过程中没有出现的doc ,乘以-100使得权重乘积最小表示当前label不适用于此doc weight_row[j] = (1 + docs_with_current_term[j] * term_doc_freq[term] * labels[i].label_weight) * (- 100) # 针对第一次没有在nonzero_index中而在后面的过程中出现的doc elif docs_with_current_term[j] == 0 and j in nonzero_index: # 加1防止权重之积为0 weight_row[j] = (1 + docs_with_current_term[j] * labels[i].label_weight) * (- 100) c += 1 else: is_incomplete = True label_term.append(terms) # bugfix:如果当前label经分词后,不是所有的term都在全部doc的term中,那么放弃当前label,舍之。 if is_incomplete: weight_row = {} for doc, weight in weight_row.items(): last = all_weight_table.get(doc) if weight > 0: if not label_doc_map.has_key(labels[i].text): kc = dao.get_keyword_category_by_category(self.query, labels[i].text) #label_doc.append([ 0,labels[i].text,[]]) label_doc.append([ 0,labels[i].text,0]) label_doc_map[labels[i].text] = len(label_doc) - 1 new_label = pextractor.Substring() new_label.text = labels[i].text new_label.id = weight if last: all_weight_table[doc].append(new_label) else: all_weight_table[doc] = [new_label] #label_doc[label_doc_map[labels[i].text]][2].append(doc) label_doc[label_doc_map[labels[i].text]][2] += 1 label_doc[label_doc_map[labels[i].text]][0] += weight #try: # category = dao.save_category(labels[i].text, weight, 'd') # entry = self.entries[doc] # ec = dao.save_entry_cat(entry, category, weight) #except Exception,e: # print e #if last: # all_weight_table[doc].append(ec) #else: # all_weight_table[doc] = [ec] # 如果doc已经存在,那么用已经存在的doc-label权重比较当前的权重,如果当前的更大则替换已经存在的,即选择最大权重的label #if last: # if last.id < weight and weight > 0: # labels[i].id = weight # all_weight_table[doc] = labels[i] #else: # labels[i].id = weight # all_weight_table[doc] = labels[i] label_doc.sort(reverse=True) for k, v in all_weight_table.items(): v.sort(reverse=True) # 因为map中键为连续的整数值,哈希算法会把他按从小到大的位置排放,所以直接返回的values是已经排好序的了 thread = SaveLabelsThread(all_weight_table,label_doc,self.entries,self.query) thread.start() return all_weight_table,label_doc """ 废弃 """ def label_assign(self, docs, labels, lucene_ids): term_row = {} all = [] ireader = IndexReader.open(STORE_DIR) total_terms = 0 for i in range(len(lucene_ids)): tpv = TermPositionVector.cast_(ireader.getTermFreqVector(lucene_ids[i], 'summary')) self.add2matrix(tpv, all, term_row, lucene_ids, i) tpv = TermPositionVector.cast_(ireader.getTermFreqVector(lucene_ids[i], 'title')) self.add2matrix(tpv, all, term_row, lucene_ids, i) # 对label进行分词 analyzer = CJKAnalyzer() labelmatrix = zeros((len(all), len(labels))) label_term = [] for i in range(len(labels)): if not labels[i].is_candicate_label and len(labels[i].text) >= 3: label_term.append([]) continue #print labels[i].text,labels[i].id stream = analyzer.tokenStream('', StringReader(labels[i].text)) terms = [] for token in stream: if term_row.has_key(token.term()): # weighting termdocs = ireader.termDocs(Term('summary', token.term())) count = 0 span = 0 terms.append(token.term()) while termdocs.next(): count += termdocs.freq() span += 1 weight = labels[i].label_weight #if float(span)/ireader.numDocs() >= 0.18 and not re.search('a-zA-z', token.term()): #weight = 0 labelmatrix[term_row[token.term()]][i] = weight label_term.append(terms) termmatrix = array(all) termmatrix = transpose(termmatrix) #for i in range(len(labelmatrix[0])): #for j in range(len(termmatrix[0])): # row是doc,col是label #p = self.product(termmatrix,labelmatrix) d = dot(termmatrix, labelmatrix) result = d / (norm(labelmatrix) * norm(termmatrix)) doc_label = [] for i in range(len(result)): m = - 1 index = - 1 group = [] for j in range(len(result[i])): if result[i][j] > 0: labels[j].id = result[i][j] group.append(labels[j]) # substring是按照id来排序的,这里正好用到 group.sort() group.reverse() max_label = group[0] # i:doc number(just occur position in the docs) # label id # label score # 如果label自身并没有出现在当前doc中 if not max_label.doc_freq.has_key(i): #print 'oringial:',labels[index].text count = 0 overlap = '' for k in label_term[index]: if term_row.has_key(k) and termmatrix[i][term_row[k]] != 0: overlap = k print k count += 1 # 至少有一个交集,并且长度大于等于2 if count == 1 and len(overlap) >= 2 : new_label = pextractor.Substring() new_label.text = overlap new_label.id = m doc_label.append(group[0]) continue #labels[index].id = m doc_label.append(group[0]) return doc_label """ 计算tf-idf值 """ def tfidf(self, terms_length, tf, docs_length, dtf): # results[i].freq - v可能为0,所以将v/2再相减 return (tf / float(terms_length)) * abs(math.log((dtf - tf / 2.0) / (docs_length*10))) def build(self, docs,query): self.entries = [] self.query = query context = Context() results, lucene_ids , categories = self.get_cs_by_lucene_doc(docs, context) results_size = len(results) doc_size = len(docs) stroplabels = {} for i in range(results_size): #删除一些类似‘在阿里巴巴’中的‘在’字,类似地还有‘...上’,‘...里’,‘和...’ stream2 = tokenizer.tokenize(results[i].text.encode('utf-8')) if len(stream2) == 2 and len(results[i].text)>=3: if (len(stream2[0].decode('utf-8')) == 1 or len(stream2[1].decode('utf-8')) == 1): results[i].text = len(stream2[0].decode('utf-8')) == 1 and stream2[1].decode('utf-8') or stream2[0].decode('utf-8') elif (len(stream2[1].decode('utf-8')) - len(stream2[0].decode('utf-8'))) == 1 \ and len(stream2[1].decode('utf-8')) == len(results[i].text): results[i].text = stream2[0].decode('utf-8') if stroplabels.has_key(results[i].text): same_label_name = stroplabels[results[i].text] results[i].freq += same_label_name.freq for k,v in same_label_name.doc_freq.items(): if results[i].doc_freq.has_key(k): results[i].doc_freq[k] += v else: results[i].doc_freq[k] = v stroplabels[results[i].text] = results[i] results = stroplabels.values() results_size = len(results) # phrase doc matrix pdmatrix = zeros((results_size, doc_size)) #for i in results: #print i.text,i.freq d = len(context.tokens) if results_size == 0: print 'no frequent phrase!' return {},[] for i in range(results_size): doc_freq = results[i].doc_freq index = context.suffix[results[i].id] # 如果有性能问题可以把title_field改为map is_title = False if index in context.title_field: is_title = True for k, v in doc_freq.items(): # k is doc_id,v is doc freq # doc_length 所在doc的term数量 doc_length = context.term_doc_range[k] - (k and context.term_doc_range[k - 1]) pdmatrix[i][k] = self.tfidf(doc_length, v, d, results[i].freq) * (is_title and TITLE_FIELD_BOOST or 1) results[i].label_weight = norm(pdmatrix[i]) #if stroplabels.has_key(results[i].text): # stroplabels[results[i].text].label_weight+=results[i].label_weight if 1: # SVD-奇异值分解 u, s, v = linalg.svd(pdmatrix) # 试探以取得矩阵的秩 rank = 1 for k in range(1, len(s)): if norm(s[:k]) / norm(s) >= CLT: rank = k break # 取u中前rank列,选择最大的一个component,保存对应的label索引 maxes = {} lo = {} for j in range(rank): m = - 100 index = - 1 for i in range(len(u)): if u[i][j] > 0: lo[i]=u[i][j] if u[i][j] > m: m = u[i][j] # index 为 对应的label索引 index = i # 如果label之前已有则取均值 if maxes.has_key(index): maxes[index] = (maxes[index] + m) / 2.0 else: # 保存最大值 maxes[index] = m labels = [] # k为索引,v为最大值 for k, v in maxes.items(): #print results[k].text,k,v #sub = pextractor.Substring() #sub.text = results[k].text temp = re.sub('[0-9 ]', '', results[k].text) if temp == '': results[k].text = temp results[k].is_candicate_label = True results[k].label_weight = v labels.append(results[k]) return self.assign(docs, results, lucene_ids)
def a(): import os #loader = BSDDictLoader() #dic = loader.load() words_dict = {} from dot.searcher import Searcher, STORE_DIR from apps.wantown import dao from apps.wantown.models import Entry searcher = Searcher() hits = searcher.search("java") docs = [] for hit in hits: doc = Hit.cast_(hit).getDocument() docs.append(doc) entries = [] all = '' from dot.context import Context, Token context = Context() import re #all = re.sub('[0-9:;;/\(\)\t\[\]]()\**#&','',all) #all = re.sub('[ +=-]',' ',all) analyzer = StandardAnalyzer() # doc id id = 0 allToken = [] allText = [] pureText = '' c = 0 docRange = {} for doc in docs[0:100]: link = doc.get("link") entry = dao.get_by_link(link, Entry) entries.append(entry.summary) all = entry.summary[:200] + entry.title pureText += all tokenType = [] last_type = '' #all = """提起电吉他演奏,就必须提到布鲁斯音乐;提起最伟大的吉他演奏大师,人们首先会想到的是 Jimi Hendrix,但是说起依然在世的最伟大的吉他演奏家,名字只有一个——Eric Clapton爵士。自从上个世纪60年代布鲁斯摇滚乐以及布鲁斯吉他演奏成为了主流摇滚风格之后,在这种来源于黑人音乐的吉他演奏中,在所有除黑色外其他肤色的布鲁斯吉他演奏家之中,传奇人物Eric Clapton毫无疑问是其中最杰出的一位。在与Eric Clapton同时代的所有艺术家纷纷的离开人世,或者失去了原有的歌迷号召力之后,Eric Clapton是所有当年这些艺术家中为数不多的既然保持着自己高超的演奏技术以及强大的市场号召力的艺术家。 # #Eric Clapton为人谦逊,在与其他出色的吉他演奏者比如Jimi Hendrix,B.B. King,Duane Allman,甚至后辈Stevie Ray Vaughan相比较的时候他总是非常谦恭,在与B.B. King以及Bob Dylan等人同台的时候他总是举止非常礼让,他是最有绅士风度的流行音乐家之一。同时,作为世界上最著名的吉他大师,Eric Clapton还经常热心的帮助包括英国著名流行音乐家Sting,Bon Jovi乐队主音吉他手Richie Sambora在内的其他一些音乐家去录制专辑或者拍摄音乐录影带,并且经常为一些音乐家担任吉他手作伴奏。Eric Clapton曾经协助过Bob Dylan,Aretha Franklin,Joe Cocker,Ringo Starr,Freddie King,Roger Waters等等近百位艺术家的专辑录制。 #""" stream = analyzer.tokenStream("fieldname", StringReader(all)) for s in stream: #if (last_type == '<ALPHANUM>' or last_type == '<HOST>') and (s.type() == '<ALPHANUM>' or s.type() == '<HOST>'): #all.append(' ') #pass #last_type = s.type() token = Token() token.text = s.termText() token.offset = s.termLength() token.doc = id allToken.append(token) allText.append(s.term()) print dir(s) c += 1 docRange[len(allText)] = id #all = sorted(all,cmp=lambda x,y:cmp(x.termText(),y.termText())) id += 1 context.tokens = allText #context.tokens.sort() #for i in context.tokens: #print i #print s context.text = '' context.token_types = tokenType context.docs = entries context.term_doc_range = docRange print len(all) from dot.lingo import pextractor import time start = time.time() #pe = pextractor.PhraseExtractor() #results = pe.extract(context) count = 0 r = docRange.keys() r.sort() if 0: for i in results: if len(i.text) > 1 and i.freq > 2 and len(i.text) < 20: id = i.id - 1 lcp = context.lcp[id + 1] for f in range(i.freq): begin = context.suffix[id] end = context.suffix[id] + lcp for j in range(len(r)): if begin < r[j]: break doc = docRange[r[j]] #print context.tokens[begin:end],i.freq,begin,doc if end > r[j]: print 'not in the same doc' id += 1 #print i.text.strip(), i.freq,i.doc_freq #print (time.time() - start) from dot.matrixmapper import MatrixMapper mapper = MatrixMapper() mapper.build(docs[:100]) #print pureText import sys from dot.lingo import suffixsorter as ss #for i in range(len(context.suffix)): # s = pe.list2str(context.tokens) # sys.stdout.write('%d\t%d\t%s\n' % (context.suffix[i], context.lcp[i], context.tokens[context.suffix[i]:context.suffix[i] + 10])) #dm = getDictManager() #words_dict= featurex.tf_idf(entries, dm.seg_dict) #doc1 = featurex.Document(entries.encode('utf-8'),dm) #doc2 = featurex.Document(entries[0].encode('utf-8'), dm) #for i in words_dict.values(): #print i.word,i.frequency,i.feature_value,i.tfidf #print similitude_doc_cos(doc1, doc2) """ ibm jdk 3 {3: 3} 不同 3 {4: 2, 7: 1} 使用 3 {8: 2, 7: 1} 可以 10 {8: 3, 3: 2, 4: 2, 7: 3} 处理 3 {8: 3} 好的 3 {8: 1, 7: 2} 字体 5 {8: 2, 4: 3} 已经 4 {9: 1, 3: 1, 4: 1, 7: 1} 平滑 4 {8: 1, 4: 3} 应用 3 {8: 1, 4: 2} 手机上 3 {7: 3} 文本 3 {8: 3} 游戏 4 {7: 4} 环境 3 {1: 1, 3: 2} 的java 6 {1: 1, 2: 1, 5: 1, 7: 3} 的文 3 {8: 3} 设置 5 {4: 5} 软件 3 {1: 1, 7: 2} 运行 3 {1: 1, 7: 2} """
# Get the analyzer analyzer = StandardAnalyzer(Version.LUCENE_CURRENT) # Constructs a query parser. queryParser = QueryParser(Version.LUCENE_CURRENT, FIELD_CONTENTS, analyzer) # Create a query query = queryParser.parse(QUERY_STRING) topDocs = searcher.search(query, 50) # Get top hits scoreDocs = topDocs.scoreDocs print "%s total matching documents." % len(scoreDocs) HighlightFormatter = SimpleHTMLFormatter() query_score = QueryScorer (query) highlighter = Highlighter(HighlightFormatter, query_score) # Set the fragment size. We break text in to fragment of 64 characters fragmenter = SimpleSpanFragmenter(query_score, 64) highlighter.setTextFragmenter(fragmenter) for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) text = doc.get(FIELD_CONTENTS) ts = analyzer.tokenStream(FIELD_CONTENTS, StringReader(text)) print doc.get(FIELD_PATH) print highlighter.getBestFragments(ts, text, 3, "...") print ""