def highlighting(analyzer,contents,query):
formatter=SimpleHTMLFormatter("<b><font color='black'>","</font></b>")
highlighter=Highlighter(formatter,QueryScorer(query))
highlighter.setTextFragmenter(SimpleFragmenter(30))
tokenStream=analyzer.tokenStream('contents',contents)
light_content=highlighter.getBestFragments(tokenStream,contents,3,'...')
return light_content
def search(self, q, page = 1, duplicates = False):
query = self.parser.parse(q)
if not duplicates:
query = self.addDuplicatesQuery(query)
perPage = 10
start = (page - 1) * perPage
results = TopScoreDocCollector.create(1000, True)
self.searcher.search(query, results)
highlighter = Highlighter(QueryScorer(query))
highlighter.setTextFragmenter(SimpleFragmenter(40))
docs = []
for scoreDoc in results.topDocs(start, perPage).scoreDocs:
doc = self.searcher.doc(scoreDoc.doc)
tokenStream = self.analyzer.tokenStream("contents", StringReader(doc['contents']))
highlight = highlighter.getBestFragments(tokenStream, doc['contents'], 3, "...")
docs.append({
'title': doc['title'],
'url': doc['url'],
'duplicate': doc['duplicate'],
'highlight': highlight}
)
del self.searcher
totalPages = int(math.ceil(results.getTotalHits()/float(perPage)))
return totalPages, docs
def text_search(command):
envir.vm_env.attachCurrentThread()
command_dict = parseCommand(command, "contents")
querys = BooleanQuery()
for k, v in command_dict.iteritems():
query = QueryParser(Version.LUCENE_CURRENT, k,
envir.analyzer).parse(v)
querys.add(query, BooleanClause.Occur.MUST)
scoreDocs = envir.text_searcher.search(querys, 30).scoreDocs
res = []
query_highlight = QueryParser(Version.LUCENE_CURRENT, k,
envir.analyzer).parse(command_dict["contents"])
myhighlighter = Highlighter(
SimpleHTMLFormatter(), QueryScorer(query_highlight))
myhighlighter.setTextFragmenter(SimpleFragmenter(50))
for scoreDoc in scoreDocs:
# find texts which are around the keyword
doc = envir.text_searcher.doc(scoreDoc.doc)
text = doc.get("contents")
key_text = "".join((myhighlighter.getBestFragments(
envir.analyzer, "contents", text, 3)))
key_text = re.sub('\s', '', key_text)
temp = [doc.get("title"), doc.get('url'), key_text]
res.append(temp)
return res
def search(self, q, page=1, duplicates=False):
query = self.parser.parse(q)
if not duplicates:
query = self.addDuplicatesQuery(query)
perPage = 10
start = (page - 1) * perPage
results = TopScoreDocCollector.create(1000, True)
self.searcher.search(query, results)
highlighter = Highlighter(QueryScorer(query))
highlighter.setTextFragmenter(SimpleFragmenter(40))
docs = []
for scoreDoc in results.topDocs(start, perPage).scoreDocs:
doc = self.searcher.doc(scoreDoc.doc)
tokenStream = self.analyzer.tokenStream(
"contents", StringReader(doc['contents']))
highlight = highlighter.getBestFragments(tokenStream,
doc['contents'], 3, "...")
docs.append({
'title': doc['title'],
'url': doc['url'],
'duplicate': doc['duplicate'],
'highlight': highlight
})
del self.searcher
totalPages = int(math.ceil(results.getTotalHits() / float(perPage)))
return totalPages, docs
def lucene_search(query, MAX, showHighlight):
dir = os.getcwd()
lucene.initVM()
index_dir = SimpleFSDirectory(File(dir))
index_reader = DirectoryReader.open(index_dir)
lucene_searcher = IndexSearcher(index_reader)
lucene_analyzer = StandardAnalyzer(Version.LUCENE_48)
my_query = QueryParser(Version.LUCENE_48, "text",
lucene_analyzer).parse(query)
#We can define the MAX number of results (default 10)
total_hits = lucene_searcher.search(my_query, MAX)
query_scorer = QueryScorer(my_query)
formatter = SimpleHTMLFormatter()
highlighter = Highlighter(formatter, query_scorer)
# Set the fragment size. We break text in to fragment of 50 characters
fragmenter = SimpleSpanFragmenter(query_scorer, 50)
highlighter.setTextFragmenter(fragmenter)
print "Only shows at most %s documents" % MAX
if showHighlight:
print "<br>"
for hit in total_hits.scoreDocs:
doc = lucene_searcher.doc(hit.doc)
text = doc.get("text")
ts = lucene_analyzer.tokenStream("text", StringReader(text))
if showHighlight:
print "<p>"
print doc.get("title")
if showHighlight:
print "<br>"
print highlighter.getBestFragments(ts, text, 3, "...")
print "</p>"
def testSimpleHighlighter(self):
self.doSearching("Wicked")
highlighter = Highlighter(QueryScorer(self.query))
highlighter.setTextFragmenter(SimpleFragmenter(40))
maxNumFragmentsRequired = 2
for scoreDoc in self.scoreDocs:
text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME)
tokenStream = self.analyzer.tokenStream(self.FIELD_NAME,
StringReader(text))
result = highlighter.getBestFragments(tokenStream, text,
maxNumFragmentsRequired,
"...")
print "\t", result
def testSimpleHighlighter(self):
self.doSearching("Wicked")
highlighter = Highlighter(QueryScorer(self.query))
highlighter.setTextFragmenter(SimpleFragmenter(40))
maxNumFragmentsRequired = 2
for scoreDoc in self.scoreDocs:
text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME)
tokenStream = self.analyzer.tokenStream(self.FIELD_NAME,
StringReader(text))
result = highlighter.getBestFragments(tokenStream, text,
maxNumFragmentsRequired,
"...")
print "\t", result
def doStandardHighlights(self):
formatter = TestFormatter(self)
highlighter = Highlighter(formatter, QueryScorer(self.query))
highlighter.setTextFragmenter(SimpleFragmenter(20))
for scoreDoc in self.scoreDocs:
text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME)
maxNumFragmentsRequired = 2
fragmentSeparator = "..."
tokenStream = self.analyzer.tokenStream(self.FIELD_NAME,
StringReader(text))
result = highlighter.getBestFragments(tokenStream, text,
maxNumFragmentsRequired,
fragmentSeparator)
print "\t", result
def doStandardHighlights(self):
formatter = TestFormatter(self)
highlighter = Highlighter(formatter, QueryScorer(self.query))
highlighter.setTextFragmenter(SimpleFragmenter(20))
for scoreDoc in self.scoreDocs:
text = self.searcher.doc(scoreDoc.doc).get(self.FIELD_NAME)
maxNumFragmentsRequired = 2
fragmentSeparator = "..."
tokenStream = self.analyzer.tokenStream(self.FIELD_NAME,
StringReader(text))
result = highlighter.getBestFragments(tokenStream,
text,
maxNumFragmentsRequired,
fragmentSeparator)
print "\t", result
def search_by(self, **kwargs):
command = kwargs.get('command', '')
if command == '':
return None
field = kwargs.get('field')
query_type = kwargs.get('query_type', 'chi')
if query_type == 'chi':
if field in ['token_taglist', 'token_content', 'token_title', 'token_author']:
command = ' '.join(jieba.cut_for_search(command))
hlt_analyzer = self.analyzer['ChineseAnalyzer']
else:
if field in ['token_content', 'token_title']:
command = ' '.join(map(stem, command.split()))
hlt_analyzer = self.analyzer['StandardAnalyzer']
analyzer = self.analyzer['SimpleAnalyzer']
num = kwargs.get('num', 50)
attrs = kwargs.get('attrs', ['url', 'title'])
print "[%s]\tSearching for '%s' in field '%s'" % (query_type, command, field)
query = QueryParser(Version.LUCENE_CURRENT, field, analyzer).parse(command)
if field in ['token_content', 'token_title']:
getAbs = True
query_for_highlight = QueryParser(Version.LUCENE_CURRENT, 'content', hlt_analyzer).parse(command)
scorer = QueryScorer(query_for_highlight)
formatter = SimpleHTMLFormatter("<strong>", "</strong>")
# formatter = SimpleHTMLFormatter("<span class=\"highlight\">", "</span>")
highlighter = Highlighter(formatter, scorer)
fragmenter = SimpleFragmenter(20)
highlighter.setTextFragmenter(fragmenter)
else:
getAbs = False
scoreDocs = self.searcher.search(query, num).scoreDocs
print "%s total matching documents." % len(scoreDocs)
articles = []
for scoreDoc in scoreDocs:
doc = self.searcher.doc(scoreDoc.doc)
article = {}
for attr in attrs:
article[attr] = doc.get(attr)
if getAbs is True:
content = doc.get('content')
tokenStream = hlt_analyzer.tokenStream("content", StringReader(content))
article['abstract'] = highlighter.getBestFragments(tokenStream, content, 3, "...")
articles.append(article)
return articles
def get_highlighted_hits(self):
extracted_fragments = []
scorer = QueryScorer(self.query)
fragmenter = SimpleSpanFragmenter(scorer, 10)
highlighter = Highlighter(self.formatter, scorer)
highlighter.setTextFragmenter(fragmenter)
for hit in self.hits.scoreDocs:
document = self.searcher.doc(hit.doc)
stream = TokenSources.getAnyTokenStream(self.index_reader, hit.doc,
'contents', self.analyzer)
best_fragments = highlighter.getBestFragments(
stream, document.get('contents'), 10)
for fragment in best_fragments:
print('fragment: ', fragment)
extracted_fragments.append((hit.doc, best_fragments))
return extracted_fragments
def get_lm_doc_snippets(query,
searcher,
qparser,
analyzer,
preprocessor,
topk=10):
"""
Fetches the topk document snippets given query, searcher and qparser and
returns (did, text) pair list
:param query:
:param searcher:
:param qparser:
:param topk:
:return:
"""
dids_text = []
query = qparser.parse(query)
scoreDocs = searcher.search(query, topk).scoreDocs
highlighter = Highlighter(QueryScorer(query))
highlighter.setTextFragmenter(SimpleFragmenter(100))
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
did = doc.get("id")
text = doc.get("raw")
token_stream = analyzer.tokenStream("raw", StringReader(text))
result = highlighter.getBestFragments(token_stream, text, 4, "... ")
text = get_parsed_text(result)
text = preprocess_text(preprocessor, [text])
text = " ".join(text)
dids_text.append((did, text))
return dids_text
def run(self, writer=None, analyzer=None):
if writer is None:
writer = self.writer
if analyzer is None:
analyzer = self.analyzer
searcher = IndexSearcher(DirectoryReader.open(\
SimpleFSDirectory.open(File(self.store_dir))))
while True:
print
print "Hit enter with no input to quit."
command = raw_input("Query:")
if command == '':
return
print "Searching for:", command
query = QueryParser(Version.LUCENE_43, "contents",
analyzer).parse(command)
# We'll just show the top 10 matching documents for now
scoreDocs = searcher.search(query, 10).scoreDocs
print "%s total matching documents." % len(scoreDocs)
# Highlight the matching text in red
highlighter = Highlighter(SimpleHTMLFormatter('<b><font color\
="red">', '</font></b>'), QueryScorer(query))
# Using NullFragmenter since we still want to see
# the whole document
highlighter.setTextFragmenter(NullFragmenter())
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
tokenStream = analyzer.tokenStream("contents",
StringReader(doc.get("contents")))
# arg 3: the maximum number of fragments
# arg 4: the separator used to intersperse the
# document fragments (typically "...")
# arg 3 and 4 don't really matter with NullFragmenter
result = highlighter.getBestFragments(tokenStream,
doc.get("contents"), 2, "...")
if len(result) > 10:
file_handler = open(self.hits_dir + '/' + doc.get("name"),
'w+')
file_handler.write(result)
# create hit fragments, if we want to show them
# arg 1: fragment size
highlighter.setTextFragmenter(SimpleFragmenter(200))
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
tokenStream = analyzer.tokenStream("contents",
StringReader(doc.get("contents")))
result = highlighter.getBestFragments(tokenStream,
doc.get("contents"), 2, "...")
if len(result) > 10:
file_handler = open(self.frags_dir + '/' + doc.get("name"),
'w+')
file_handler.write(result)
def _getHits(self):
reader = IndexReader.open(self.index)
searcher = IndexSearcher(reader)
# Sortierung nach Band- und Eintragsnummer (4: Wert als Integer behandeln)
sortDict = {
"ta": (("volume", SortField.Type.INT), ("number", SortField.Type.INT)),
"year": (("year-start", SortField.Type.INT), ("year-end", SortField.Type.INT)),
"author-title": (("author-sort", SortField.Type.STRING), ("title-sort", SortField.Type.STRING)),
"title": (("title-sort", Locale.GERMAN),),
"author": (("author-sort", Locale.GERMAN),),
}
sortFields = []
reverse = not self.ascending
for name in self.orderby:
for fieldName, typeNum in sortDict.get(name, []):
sortFields.append(SortField(fieldName, typeNum, reverse))
if len(sortFields) == 0:
sortFields = [SortField("volume", SortField.Type.INT), SortField("number", SortField.Type.INT)]
sort = Sort(sortFields)
topDocs = searcher.search(self.query, None, 80000, sort)
hits = topDocs.scoreDocs
self.hits = hits
self.searcher = searcher
lang = translation.get_language()
if lang != "de":
lang = "en"
facets = {"author": {}, "pubtype": {}, "category-%s" % lang: {}}
# Highlighting
highlighter = Highlighter(SimpleHTMLFormatter('<span class="highlight">', '</span>'), QueryScorer(self.query))
hitObjects = []
fields = {}
for hit in hits:
doc = searcher.doc(hit.doc)
# print unicode(doc)
fields["score"] = hit.score
fields["volume"] = doc["volume"]
fields["number"] = doc["number"]
fields["id"] = doc["id"]
fields["title"] = doc["title"]
fields["author"] = doc["author"]
fields["authors"] = [field.stringValue() for field in doc.getFields("author")]
for author in fields["authors"]: # XXX
facets["author"][author] = facets["author"].get(author, 0) + 1 # XXX
fields["categories"] = [field.stringValue() for field in doc.getFields("category-%s" % lang)]
for cat in fields["categories"]:
facets["category-%s" % lang][cat] = facets["category-%s" % lang].get(cat, 0) + 1
maxNumFragmentsRequired = 2
fragmentSeparator = "...";
pubtype = doc["pubtype"]
fields["pubtype"] = pubtype
facets["pubtype"][pubtype] = facets["pubtype"].get(pubtype, 0) + 1
fields["city"] = doc["city"]
fields["year"] = doc["year-start"]
if fields["year"] and doc["year-end"] and doc["year-end"] != fields["year"]:
fields["year"] += " - " + doc["year-end"]
highlightFields = ("title", "author", "city", "year", "category")
if "freetext" in self.criteria:
for fieldName in highlightFields:
try:
tokenStream = self.analyzer.tokenStream(fieldName, lucene.StringReader(fields[fieldName]))
newVal = highlighter.getBestFragments(tokenStream, fields[fieldName], maxNumFragmentsRequired,
fragmentSeparator)
if len(newVal) > 0:
# fields[fieldName] = re.sub(r'</span>\s*<span class="highlight">', ' ', newVal)
fields[fieldName] = newVal
except:
continue
for fieldName in highlightFields:
if fieldName in self.criteria or fieldName + "-de" in self.criteria or fieldName + "-en" in self.criteria:
try:
tokenStream = self.analyzer.tokenStream(fieldName, lucene.StringReader(fields[fieldName]))
newVal = highlighter.getBestFragments(tokenStream, fields[fieldName], maxNumFragmentsRequired,
fragmentSeparator)
if len(newVal) > 0:
# fields[fieldName] = re.sub(r'</span>\s*<span class="highlight">', ' ', newVal)
fields[fieldName] = newVal
except:
continue
"""if "author" in self.criteria:
try:
tokenStream = self.analyzer.tokenStream("author", lucene.StringReader(fields["author"]))
fields["author"] = highlighter.getBestFragments(tokenStream, fields["author"], maxNumFragmentsRequired, fragmentSeparator)
except:
pass"""
hitObjects.append(
Hit(fields["id"], fields["volume"], fields["number"], fields["title"], fields["author"], fields["city"],
fields["year"], fields["categories"], fields["pubtype"], fields["score"]))
facetsToDelete = []
for facet in facets:
if len(facets[facet]) < 2:
facetsToDelete.append(facet)
continue
values = sorted(facets[facet].items(), key=itemgetter(0))
values = sorted(values, key=itemgetter(1), reverse=True)
facets[facet] = values[:25]
for facet in facetsToDelete:
del facets[facet]
self.facets = facets
reader.close()
self.hitObjects = hitObjects
return hitObjects
def run(self, writer=None, analyzer=None):
if writer is None:
writer = self.writer
if analyzer is None:
analyzer = self.analyzer
searcher = IndexSearcher(DirectoryReader.open(\
SimpleFSDirectory.open(File(self.store_dir))))
while True:
print()
print("Hit enter with no input to quit.")
command = input("Query:")
if command == '':
return
print("Searching for:", command)
query = QueryParser(Version.LUCENE_43, "contents",
analyzer).parse(command)
# We'll just show the top 10 matching documents for now
scoreDocs = searcher.search(query, 10).scoreDocs
print("%s total matching documents." % len(scoreDocs))
# Highlight the matching text in red
highlighter = Highlighter(
SimpleHTMLFormatter('<b><font color\
="red">', '</font></b>'), QueryScorer(query))
# Using NullFragmenter since we still want to see
# the whole document
highlighter.setTextFragmenter(NullFragmenter())
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
tokenStream = analyzer.tokenStream(
"contents", StringReader(doc.get("contents")))
# arg 3: the maximum number of fragments
# arg 4: the separator used to intersperse the
# document fragments (typically "...")
# arg 3 and 4 don't really matter with NullFragmenter
result = highlighter.getBestFragments(tokenStream,
doc.get("contents"), 2,
"...")
if len(result) > 10:
file_handler = open(self.hits_dir + '/' + doc.get("name"),
'w+')
file_handler.write(result)
# create hit fragments, if we want to show them
# arg 1: fragment size
highlighter.setTextFragmenter(SimpleFragmenter(200))
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
tokenStream = analyzer.tokenStream(
"contents", StringReader(doc.get("contents")))
result = highlighter.getBestFragments(tokenStream,
doc.get("contents"), 2,
"...")
if len(result) > 10:
file_handler = open(self.frags_dir + '/' + doc.get("name"),
'w+')
file_handler.write(result)