def text_search(command): envir.vm_env.attachCurrentThread() command_dict = parseCommand(command, "contents") querys = BooleanQuery() for k, v in command_dict.iteritems(): query = QueryParser(Version.LUCENE_CURRENT, k, envir.analyzer).parse(v) querys.add(query, BooleanClause.Occur.MUST) scoreDocs = envir.text_searcher.search(querys, 30).scoreDocs res = [] query_highlight = QueryParser(Version.LUCENE_CURRENT, k, envir.analyzer).parse(command_dict["contents"]) myhighlighter = Highlighter( SimpleHTMLFormatter(), QueryScorer(query_highlight)) myhighlighter.setTextFragmenter(SimpleFragmenter(50)) for scoreDoc in scoreDocs: # find texts which are around the keyword doc = envir.text_searcher.doc(scoreDoc.doc) text = doc.get("contents") key_text = "".join((myhighlighter.getBestFragments( envir.analyzer, "contents", text, 3))) key_text = re.sub('\s', '', key_text) temp = [doc.get("title"), doc.get('url'), key_text] res.append(temp) return res
def highlighting(analyzer,contents,query): formatter=SimpleHTMLFormatter("<b><font color='black'>","</font></b>") highlighter=Highlighter(formatter,QueryScorer(query)) highlighter.setTextFragmenter(SimpleFragmenter(30)) tokenStream=analyzer.tokenStream('contents',contents) light_content=highlighter.getBestFragments(tokenStream,contents,3,'...') return light_content
def search(self, q, page=1, duplicates=False): query = self.parser.parse(q) if not duplicates: query = self.addDuplicatesQuery(query) perPage = 10 start = (page - 1) * perPage results = TopScoreDocCollector.create(1000, True) self.searcher.search(query, results) highlighter = Highlighter(QueryScorer(query)) highlighter.setTextFragmenter(SimpleFragmenter(40)) docs = [] for scoreDoc in results.topDocs(start, perPage).scoreDocs: doc = self.searcher.doc(scoreDoc.doc) tokenStream = self.analyzer.tokenStream( "contents", StringReader(doc['contents'])) highlight = highlighter.getBestFragments(tokenStream, doc['contents'], 3, "...") docs.append({ 'title': doc['title'], 'url': doc['url'], 'duplicate': doc['duplicate'], 'highlight': highlight }) del self.searcher totalPages = int(math.ceil(results.getTotalHits() / float(perPage))) return totalPages, docs
def search(self, q, page = 1, duplicates = False): query = self.parser.parse(q) if not duplicates: query = self.addDuplicatesQuery(query) perPage = 10 start = (page - 1) * perPage results = TopScoreDocCollector.create(1000, True) self.searcher.search(query, results) highlighter = Highlighter(QueryScorer(query)) highlighter.setTextFragmenter(SimpleFragmenter(40)) docs = [] for scoreDoc in results.topDocs(start, perPage).scoreDocs: doc = self.searcher.doc(scoreDoc.doc) tokenStream = self.analyzer.tokenStream("contents", StringReader(doc['contents'])) highlight = highlighter.getBestFragments(tokenStream, doc['contents'], 3, "...") docs.append({ 'title': doc['title'], 'url': doc['url'], 'duplicate': doc['duplicate'], 'highlight': highlight} ) del self.searcher totalPages = int(math.ceil(results.getTotalHits()/float(perPage))) return totalPages, docs
def output(self, score_docs, command): ''' Highlight and return the search results. Input: `score_docs`: search results from the index Output: list of documents info found in the index, details includes `title`, `url` and `abstract` ''' query = QueryParser('contents', self.analyzer).parse(command) highlighter = Highlighter(self.formatter, QueryScorer(query)) highlighter.setTextFragmenter( SimpleFragmenter(200)) # Limit the max number of characters results = [] for score_doc in score_docs: doc = self.searcher.doc(score_doc.doc) contents = doc.get('contents') stream = self.analyzer.tokenStream("contents", contents) abstract = highlighter.getBestFragment( stream, contents) # Get the abstract and highlight result = { 'title': doc.get('title'), 'url': doc.get('url'), 'abstract': abstract.replace(' ', '') } results.append(result) return results
def testSimpleHighlighter(self): self.doSearching("Wicked") highlighter = Highlighter(QueryScorer(self.query)) highlighter.setTextFragmenter(SimpleFragmenter(40)) maxNumFragmentsRequired = 2 for scoreDoc in self.scoreDocs: text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME) tokenStream = self.analyzer.tokenStream(self.FIELD_NAME, StringReader(text)) result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...") print "\t", result
def testSimpleHighlighter(self): self.doSearching("Wicked") highlighter = Highlighter(QueryScorer(self.query)) highlighter.setTextFragmenter(SimpleFragmenter(40)) maxNumFragmentsRequired = 2 for scoreDoc in self.scoreDocs: text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME) tokenStream = self.analyzer.tokenStream(self.FIELD_NAME, StringReader(text)) result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...") print "\t", result
def doStandardHighlights(self): formatter = TestFormatter(self) highlighter = Highlighter(formatter, QueryScorer(self.query)) highlighter.setTextFragmenter(SimpleFragmenter(20)) for scoreDoc in self.scoreDocs: text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME) maxNumFragmentsRequired = 2 fragmentSeparator = "..." tokenStream = self.analyzer.tokenStream(self.FIELD_NAME, StringReader(text)) result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, fragmentSeparator) print "\t", result
def doStandardHighlights(self): formatter = TestFormatter(self) highlighter = Highlighter(formatter, QueryScorer(self.query)) highlighter.setTextFragmenter(SimpleFragmenter(20)) for scoreDoc in self.scoreDocs: text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME) maxNumFragmentsRequired = 2 fragmentSeparator = "..." tokenStream = self.analyzer.tokenStream(self.FIELD_NAME, StringReader(text)) result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, fragmentSeparator) print "\t", result
def search_by(self, **kwargs): command = kwargs.get('command', '') if command == '': return None field = kwargs.get('field') query_type = kwargs.get('query_type', 'chi') if query_type == 'chi': if field in ['token_taglist', 'token_content', 'token_title', 'token_author']: command = ' '.join(jieba.cut_for_search(command)) hlt_analyzer = self.analyzer['ChineseAnalyzer'] else: if field in ['token_content', 'token_title']: command = ' '.join(map(stem, command.split())) hlt_analyzer = self.analyzer['StandardAnalyzer'] analyzer = self.analyzer['SimpleAnalyzer'] num = kwargs.get('num', 50) attrs = kwargs.get('attrs', ['url', 'title']) print "[%s]\tSearching for '%s' in field '%s'" % (query_type, command, field) query = QueryParser(Version.LUCENE_CURRENT, field, analyzer).parse(command) if field in ['token_content', 'token_title']: getAbs = True query_for_highlight = QueryParser(Version.LUCENE_CURRENT, 'content', hlt_analyzer).parse(command) scorer = QueryScorer(query_for_highlight) formatter = SimpleHTMLFormatter("<strong>", "</strong>") # formatter = SimpleHTMLFormatter("<span class=\"highlight\">", "</span>") highlighter = Highlighter(formatter, scorer) fragmenter = SimpleFragmenter(20) highlighter.setTextFragmenter(fragmenter) else: getAbs = False scoreDocs = self.searcher.search(query, num).scoreDocs print "%s total matching documents." % len(scoreDocs) articles = [] for scoreDoc in scoreDocs: doc = self.searcher.doc(scoreDoc.doc) article = {} for attr in attrs: article[attr] = doc.get(attr) if getAbs is True: content = doc.get('content') tokenStream = hlt_analyzer.tokenStream("content", StringReader(content)) article['abstract'] = highlighter.getBestFragments(tokenStream, content, 3, "...") articles.append(article) return articles
def get_highlighted_hits(self): extracted_fragments = [] scorer = QueryScorer(self.query) fragmenter = SimpleSpanFragmenter(scorer, 10) highlighter = Highlighter(self.formatter, scorer) highlighter.setTextFragmenter(fragmenter) for hit in self.hits.scoreDocs: document = self.searcher.doc(hit.doc) stream = TokenSources.getAnyTokenStream(self.index_reader, hit.doc, 'contents', self.analyzer) best_fragments = highlighter.getBestFragments( stream, document.get('contents'), 10) for fragment in best_fragments: print('fragment: ', fragment) extracted_fragments.append((hit.doc, best_fragments)) return extracted_fragments
def run(searcher, analyzer, command): command_dict = parseCommand(command) seg_list = jieba.cut(command_dict['contents']) command_dict['contents'] = (" ".join(seg_list)) querys = BooleanQuery() for k, v in command_dict.iteritems(): query = QueryParser(Version.LUCENE_CURRENT, k, analyzer).parse(v) querys.add(query, BooleanClause.Occur.MUST) scoreDocs = searcher.search(querys, 50).scoreDocs print "%s total matching documents." % len(scoreDocs) scorer = QueryScorer(query) fragmenter = SimpleSpanFragmenter(scorer, 250) simpleHTMLFormatter = SimpleHTMLFormatter("<b>", "</b>") highlighter = Highlighter(simpleHTMLFormatter, scorer) highlighter.setTextFragmenter(fragmenter) results = [] for i, scoreDoc in enumerate(scoreDocs): doc = searcher.doc(scoreDoc.doc) contents = doc.get("contents") if contents: tkStream = analyzer.tokenStream("contents", contents) highlight = highlighter.getBestFragment(tkStream, contents) highlightseg = highlight.split() highlight = ''.join(highlightseg) results.append( (doc.get("title").strip(), doc.get("url"), highlight)) ''' print 'path:', doc.get("path"), \ '\nname:', doc.get("name"), \ '\ntitle:', doc.get("title"), \ "url:",doc.get("url"), \ "\nsite:",doc.get("site"),\ "\ncontent:",highlight,"\n" ''' # print 'explain:', searcher.explain(query, scoreDoc.doc) return results
def get_lm_doc_snippets(query, searcher, qparser, analyzer, preprocessor, topk=10): """ Fetches the topk document snippets given query, searcher and qparser and returns (did, text) pair list :param query: :param searcher: :param qparser: :param topk: :return: """ dids_text = [] query = qparser.parse(query) scoreDocs = searcher.search(query, topk).scoreDocs highlighter = Highlighter(QueryScorer(query)) highlighter.setTextFragmenter(SimpleFragmenter(100)) for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) did = doc.get("id") text = doc.get("raw") token_stream = analyzer.tokenStream("raw", StringReader(text)) result = highlighter.getBestFragments(token_stream, text, 4, "... ") text = get_parsed_text(result) text = preprocess_text(preprocessor, [text]) text = " ".join(text) dids_text.append((did, text)) return dids_text
def lucene_search(query, MAX, showHighlight): dir = os.getcwd() lucene.initVM() index_dir = SimpleFSDirectory(File(dir)) index_reader = DirectoryReader.open(index_dir) lucene_searcher = IndexSearcher(index_reader) lucene_analyzer = StandardAnalyzer(Version.LUCENE_48) my_query = QueryParser(Version.LUCENE_48, "text", lucene_analyzer).parse(query) #We can define the MAX number of results (default 10) total_hits = lucene_searcher.search(my_query, MAX) query_scorer = QueryScorer(my_query) formatter = SimpleHTMLFormatter() highlighter = Highlighter(formatter, query_scorer) # Set the fragment size. We break text in to fragment of 50 characters fragmenter = SimpleSpanFragmenter(query_scorer, 50) highlighter.setTextFragmenter(fragmenter) print "Only shows at most %s documents" % MAX if showHighlight: print "<br>" for hit in total_hits.scoreDocs: doc = lucene_searcher.doc(hit.doc) text = doc.get("text") ts = lucene_analyzer.tokenStream("text", StringReader(text)) if showHighlight: print "<p>" print doc.get("title") if showHighlight: print "<br>" print highlighter.getBestFragments(ts, text, 3, "...") print "</p>"
def run(self, writer=None, analyzer=None): if writer is None: writer = self.writer if analyzer is None: analyzer = self.analyzer searcher = IndexSearcher(DirectoryReader.open(\ SimpleFSDirectory.open(File(self.store_dir)))) while True: print print "Hit enter with no input to quit." command = raw_input("Query:") if command == '': return print "Searching for:", command query = QueryParser(Version.LUCENE_43, "contents", analyzer).parse(command) # We'll just show the top 10 matching documents for now scoreDocs = searcher.search(query, 10).scoreDocs print "%s total matching documents." % len(scoreDocs) # Highlight the matching text in red highlighter = Highlighter(SimpleHTMLFormatter('<b><font color\ ="red">', '</font></b>'), QueryScorer(query)) # Using NullFragmenter since we still want to see # the whole document highlighter.setTextFragmenter(NullFragmenter()) for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) tokenStream = analyzer.tokenStream("contents", StringReader(doc.get("contents"))) # arg 3: the maximum number of fragments # arg 4: the separator used to intersperse the # document fragments (typically "...") # arg 3 and 4 don't really matter with NullFragmenter result = highlighter.getBestFragments(tokenStream, doc.get("contents"), 2, "...") if len(result) > 10: file_handler = open(self.hits_dir + '/' + doc.get("name"), 'w+') file_handler.write(result) # create hit fragments, if we want to show them # arg 1: fragment size highlighter.setTextFragmenter(SimpleFragmenter(200)) for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) tokenStream = analyzer.tokenStream("contents", StringReader(doc.get("contents"))) result = highlighter.getBestFragments(tokenStream, doc.get("contents"), 2, "...") if len(result) > 10: file_handler = open(self.frags_dir + '/' + doc.get("name"), 'w+') file_handler.write(result)
def run(searcher, analyzer, command, urlclick): if command == '': return [] res = firstsearch(searcher, analyzer, command) command = ''.join(my_jieba.cut(command)) command = " ".join(jieba.cut(command, cut_all=True)) if len(res) > 0: scoreDocs = res else: querys = BooleanQuery() for k in tag: query = QueryParser(Version.LUCENE_CURRENT, k, analyzer).parse(command) if k == 'taste' or k == 'tech': query.setBoost(0.5) querys.add(query, BooleanClause.Occur.SHOULD) scoreDocs = searcher.search(querys, 10000).scoreDocs swxc_res = findres(command, scoreDocs, searcher) formatter = SimpleHTMLFormatter("<span style='color:red'>", "</span>") formatter_name = SimpleHTMLFormatter("<span,style='color:red'>", "</span>") scorer = QueryScorer( QueryParser(Version.LUCENE_CURRENT, 'name', analyzer).parse(command)) highlighter1 = Highlighter(formatter_name, scorer) highlighter2 = Highlighter( formatter_name, QueryScorer( QueryParser(Version.LUCENE_CURRENT, 'content', analyzer).parse(command))) highlighter3 = Highlighter( formatter, QueryScorer( QueryParser(Version.LUCENE_CURRENT, 'ingredient', analyzer).parse(command))) highlighter4 = Highlighter( formatter, QueryScorer( QueryParser(Version.LUCENE_CURRENT, 'taste', analyzer).parse(command))) highlighter5 = Highlighter( formatter, QueryScorer( QueryParser(Version.LUCENE_CURRENT, 'tech', analyzer).parse(command))) highlighter6 = Highlighter( formatter, QueryScorer( QueryParser(Version.LUCENE_CURRENT, 'others', analyzer).parse(command))) fragmenter = SimpleFragmenter(1000) highlighter1.setTextFragmenter(fragmenter) highlighter2.setTextFragmenter(fragmenter) highlighter3.setTextFragmenter(fragmenter) highlighter4.setTextFragmenter(fragmenter) highlighter5.setTextFragmenter(fragmenter) highlighter6.setTextFragmenter(fragmenter) results = [] for scoreDoc in scoreDocs: if (len(scoreDocs) > 200 and len(scoreDocs) * scoreDoc.score < 2) or scoreDoc.score < 0.002: continue doc = searcher.doc(scoreDoc.doc) highlighterContent = highlighter1.getBestFragment( analyzer, 'name', doc.get('name')) highlighterContent2 = highlighter2.getBestFragment( analyzer, 'content', doc.get('content')) highlighterContent3 = highlighter3.getBestFragment( analyzer, 'ingredient', doc.get('ingredient')) highlighterContent4 = highlighter4.getBestFragment( analyzer, 'taste', doc.get('taste')) highlighterContent5 = highlighter5.getBestFragment( analyzer, 'tech', doc.get('tech')) highlighterContent6 = highlighter6.getBestFragment( analyzer, 'others', doc.get('others')) if highlighterContent: highlighterContent = highlighterContent.replace(' ', '') highlighterContent = highlighterContent.replace(',', ' ') else: highlighterContent = doc.get('name').replace(' ', '') if highlighterContent2: highlighterContent2 = highlighterContent2.replace(' ', '') highlighterContent2 = highlighterContent2.replace(',', ' ') else: highlighterContent2 = doc.get('content').replace(' ', '') if highlighterContent3: highlighterContent3 = highlighterContent3.replace(',', '') else: highlighterContent3 = (doc.get('ingredient')).replace(',', '') if highlighterContent4: pass else: highlighterContent4 = doc.get('taste') if highlighterContent5: pass else: highlighterContent5 = doc.get('tech') if highlighterContent6: highlighterContent6 = highlighterContent6.replace(',', '') else: highlighterContent6 = (doc.get('others')).replace(',', '') results.append( (highlighterContent, doc.get('img'), highlighterContent2, highlighterContent3, highlighterContent4, highlighterContent5, highlighterContent6, doc.get('url'), scoreDoc.score)) for i in range(0, min(20, len(results)) - 1): flag = True for j in range(0, min(20, len(results)) - i - 1): if abs(results[j][8] - results[j + 1][8]) < 0.1 and urlclick[ results[j][7]] < urlclick[results[j + 1][7]]: flag = False results[j], results[j + 1] = results[j + 1], results[j] if flag: break return results, swxc_res
def run(self, writer=None, analyzer=None): if writer is None: writer = self.writer if analyzer is None: analyzer = self.analyzer searcher = IndexSearcher(DirectoryReader.open(\ SimpleFSDirectory.open(File(self.store_dir)))) while True: print() print("Hit enter with no input to quit.") command = input("Query:") if command == '': return print("Searching for:", command) query = QueryParser(Version.LUCENE_43, "contents", analyzer).parse(command) # We'll just show the top 10 matching documents for now scoreDocs = searcher.search(query, 10).scoreDocs print("%s total matching documents." % len(scoreDocs)) # Highlight the matching text in red highlighter = Highlighter( SimpleHTMLFormatter('<b><font color\ ="red">', '</font></b>'), QueryScorer(query)) # Using NullFragmenter since we still want to see # the whole document highlighter.setTextFragmenter(NullFragmenter()) for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) tokenStream = analyzer.tokenStream( "contents", StringReader(doc.get("contents"))) # arg 3: the maximum number of fragments # arg 4: the separator used to intersperse the # document fragments (typically "...") # arg 3 and 4 don't really matter with NullFragmenter result = highlighter.getBestFragments(tokenStream, doc.get("contents"), 2, "...") if len(result) > 10: file_handler = open(self.hits_dir + '/' + doc.get("name"), 'w+') file_handler.write(result) # create hit fragments, if we want to show them # arg 1: fragment size highlighter.setTextFragmenter(SimpleFragmenter(200)) for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) tokenStream = analyzer.tokenStream( "contents", StringReader(doc.get("contents"))) result = highlighter.getBestFragments(tokenStream, doc.get("contents"), 2, "...") if len(result) > 10: file_handler = open(self.frags_dir + '/' + doc.get("name"), 'w+') file_handler.write(result)
def build_highlighter(parsed_query): scorer = QueryScorer(parsed_query, 'content') highlighter = Highlighter(SimpleHTMLFormatter(), scorer) fragmenter = SimpleSpanFragmenter(scorer, FRAGMENT_SIZE) highlighter.setTextFragmenter(fragmenter) return highlighter