def indexReader(self, indexWriter, reader, uItem, uAttr, uValue, version):
STORED = Field.Store.YES
UN_INDEXED = Field.Index.NO
UN_TOKENIZED = Field.Index.UN_TOKENIZED
doc = Document()
doc.add(Field("item", uItem.str64(), STORED, UN_TOKENIZED))
doc.add(Field("attribute", uAttr.str64(), STORED, UN_TOKENIZED))
doc.add(Field("value", uValue.str64(), STORED, UN_INDEXED))
doc.add(Field("version", str(version), STORED, UN_INDEXED))
reader = StringReader(reader.read())
doc.add(Field("contents", reader, Field.TermVector.YES))
indexWriter.addDocument(doc)
def indexReader(self, indexWriter, reader, uItem, uAttr, uValue, version):
STORED = Field.Store.YES
UN_INDEXED = Field.Index.NO
UN_TOKENIZED = Field.Index.UN_TOKENIZED
doc = Document()
doc.add(Field("item", uItem.str64(), STORED, UN_TOKENIZED))
doc.add(Field("attribute", uAttr.str64(), STORED, UN_TOKENIZED))
doc.add(Field("value", uValue.str64(), STORED, UN_INDEXED))
doc.add(Field("version", str(version), STORED, UN_INDEXED))
reader = StringReader(reader.read())
doc.add(Field("contents", reader, Field.TermVector.YES))
indexWriter.addDocument(doc)
def main(cls, argv):
query = TermQuery(Term("f", "ipsum"))
scorer = QueryScorer(query)
formatter = SimpleHTMLFormatter("<span class=\"highlight\">",
"</span>")
highlighter = Highlighter(formatter, scorer)
fragmenter = SimpleFragmenter(50)
highlighter.setTextFragmenter(fragmenter)
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
tokenStream = analyzer.tokenStream("f", StringReader(cls.text))
result = highlighter.getBestFragments(tokenStream, cls.text, 5, "...")
stdout.write("<html>")
stdout.write("<style>\n")
stdout.write(".highlight {\n")
stdout.write(" background: yellow\n")
stdout.write("}\n")
stdout.write("</style>")
stdout.write("<body>")
stdout.write(result)
stdout.write("</body></html>\n")
stdout.flush()
def displayTokens(cls, analyzer, text):
tokenStream = analyzer.tokenStream("contents", StringReader(text))
term = tokenStream.addAttribute(TermAttribute.class_)
while tokenStream.incrementToken():
print "[%s]" % (term.term()),
def testHighlighting(self):
text = "The quick brown fox jumps over the lazy dog"
query = TermQuery(Term("field", "fox"))
scorer = QueryScorer(query)
highlighter = Highlighter(scorer)
tokenStream = SimpleAnalyzer(Version.LUCENE_CURRENT).tokenStream(
"field", StringReader(text))
self.assertEqual("The quick brown <B>fox</B> jumps over the lazy dog",
highlighter.getBestFragment(tokenStream, text))
def search_image(command):
if command == ' ':
return []
Docs = []
vm_env = getVMEnv()
vm_env.attachCurrentThread()
STORE_DIR = "index_img"
directory = SimpleFSDirectory(File(STORE_DIR))
searcher = IndexSearcher(directory, True)
analyzer = WhitespaceAnalyzer(Version.LUCENE_CURRENT)
command_dict = {}
allowed_opt = ['site']
opt = 'contents'
for i in command.split(' '):
if ':' in i:
opt, value = i.split(':')[:2]
opt = opt.lower()
if opt in allowed_opt and value != '':
command_dict[opt] = command_dict.get(opt, '') + ' ' + value
else:
seg_list = jieba.cut(i)
command_dict[opt] = command_dict.get(opt,
'') + ' ' + " ".join(seg_list)
querys = BooleanQuery()
for k, v in command_dict.iteritems():
if k == 'site':
t = Term(k, '*' + v)
query = WildcardQuery(t)
else:
query = QueryParser(Version.LUCENE_CURRENT, k, analyzer).parse(v)
querys.add(query, BooleanClause.Occur.MUST)
scoreDocs = searcher.search(querys, 10000).scoreDocs
formatter = SimpleHTMLFormatter("<font color=#FF0000>", "</font>")
highlighter = Highlighter(formatter, QueryScorer(querys))
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
doc_dic = {}
doc_dic["url"] = doc.get("url")
doc_dic["imgurl"] = doc.get("imgurl")
doc_dic["urltitle"] = doc.get("urltitle")
text = doc.get("contents")
ts = analyzer.tokenStream(doc.get("contents"), StringReader(text))
doc_dic["contents"] = highlighter.getBestFragments(ts, text, 2, "...")
Docs.append(doc_dic)
searcher.close()
return Docs
def assertAnalyzesTo(cls, analyzer, input, outputs):
stream = analyzer.tokenStream("field", StringReader(input))
termAttr = stream.addAttribute(TermAttribute.class_)
for output in outputs:
if not stream.incrementToken():
raise AssertionError, 'stream.incremementToken()'
if output != termAttr.term():
raise AssertionError, 'output == termAttr.term())'
if stream.incrementToken():
raise AssertionError, 'not stream.incremementToken()'
stream.close()
def dumpSpans(self, query):
spans = query.getSpans(self.reader)
print "%s:" % query
numSpans = 0
scoreDocs = self.searcher.search(query, 50).scoreDocs
scores = [0, 0]
for scoreDoc in scoreDocs:
scores[scoreDoc.doc] = scoreDoc.score
while spans.next():
numSpans += 1
id = spans.doc()
doc = self.reader.document(id)
# for simplicity - assume tokens are in sequential,
# positions, starting from 0
stream = self.analyzer.tokenStream("contents",
StringReader(doc.get("f")))
term = stream.addAttribute(TermAttribute.class_)
buffer = StringIO()
buffer.write(" ")
i = 0
while stream.incrementToken():
if i == spans.start():
buffer.write("<")
buffer.write(term.term())
if i + 1 == spans.end():
buffer.write(">")
buffer.write(" ")
i += 1
buffer.write("(")
buffer.write(str(scores[id]))
buffer.write(") ")
print buffer.getvalue()
# print self.searcher.explain(query, id)
if numSpans == 0:
print " No spans"
print ''
def displayTokensWithPositions(cls, analyzer, text):
stream = analyzer.tokenStream("contents", StringReader(text))
term = stream.addAttribute(TermAttribute.class_)
posIncr = stream.addAttribute(PositionIncrementAttribute.class_)
position = 0
while stream.incrementToken():
increment = posIncr.getPositionIncrement()
if increment > 0:
position = position + increment
print "\n%d:" % (position),
print "[%s]" % (term.term()),
print
def indexFile(self, writer, path):
try:
file = open(path)
string = HTMLReader(InputStreamReader(file, 'utf-8')).read()
file.close()
except:
raise
else:
doc = Document()
doc.add(Field("contents", StringReader(string)))
doc.add(
Field("filename", os.path.abspath(path), Field.Store.YES,
Field.Index.NOT_ANALYZED))
writer.addDocument(doc)
return doc
def testHits(self):
searcher = IndexSearcher(self.directory, True)
query = TermQuery(Term("title", "action"))
scoreDocs = searcher.search(query, 50).scoreDocs
scorer = QueryScorer(query)
highlighter = Highlighter(scorer)
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
title = doc["title"]
stream = SimpleAnalyzer(Version.LUCENE_CURRENT).tokenStream(
"title", StringReader(title))
fragment = highlighter.getBestFragment(stream, title)
print fragment
def displayTokensWithFullDetails(cls, analyzer, text):
stream = analyzer.tokenStream("contents", StringReader(text))
term = stream.addAttribute(TermAttribute.class_)
posIncr = stream.addAttribute(PositionIncrementAttribute.class_)
offset = stream.addAttribute(OffsetAttribute.class_)
type = stream.addAttribute(TypeAttribute.class_)
position = 0
while stream.incrementToken():
increment = posIncr.getPositionIncrement()
if increment > 0:
position = position + increment
print "\n%d:" % (position),
print "[%s:%d->%d:%s]" % (term.term(), offset.startOffset(),
offset.endOffset(), type.type()),
print
def indexFile(self, writer, path):
doc = Document()
try:
process = popen2.Popen4(["antiword", "-m", "UTF-8", path])
string = InputStreamReader(process.fromchild, 'utf-8').read()
except:
raise
else:
doc.add(Field("contents", StringReader(string)))
doc.add(Field("filename", os.path.abspath(path),
Field.Store.YES, Field.Index.NOT_ANALYZED))
writer.addDocument(doc)
exitCode = process.wait()
if exitCode != 0:
raise RuntimeError, "pdftotext exit code %d" %(exitCode)
return doc
def testJumps(self):
stream = self.synonymAnalyzer.tokenStream("contents",
StringReader("jumps"))
term = stream.addAttribute(TermAttribute.class_)
posIncr = stream.addAttribute(PositionIncrementAttribute.class_)
i = 0
expected = ["jumps", "hops", "leaps"]
while stream.incrementToken():
self.assertEqual(expected[i], term.term())
if i == 0:
expectedPos = 1
else:
expectedPos = 0
self.assertEqual(expectedPos, posIncr.getPositionIncrement())
i += 1
self.assertEqual(3, i)
def indexFile(self, writer, path):
doc = Document()
try:
process = popen2.Popen4(["pdfinfo", "-enc", "UTF-8", path])
except:
raise
else:
while True:
line = process.fromchild.readline().strip()
if not line:
break
name, value = line.split(':', 1)
doc.add(
Field(name.strip(), value.strip(), Field.Store.YES,
Field.Index.NOT_ANALYZED))
exitCode = process.wait()
if exitCode != 0:
raise RuntimeError, "pdfinfo exit code %d" % (exitCode)
try:
process = popen2.Popen4(["pdftotext", "-enc", "UTF-8", path, "-"])
string = InputStreamReader(process.fromchild, 'utf-8').read()
except:
raise
else:
doc.add(Field("contents", StringReader(string)))
doc.add(
Field("filename", os.path.abspath(path), Field.Store.YES,
Field.Index.NOT_ANALYZED))
writer.addDocument(doc)
exitCode = process.wait()
if exitCode != 0:
raise RuntimeError, "pdftotext exit code %d" % (exitCode)
return doc
def search(request):
vm_env = lucene.getVMEnv()
vm_env.attachCurrentThread()
ret = {}
maxLength = 38
search_content = request.GET.get('content')
if len(search_content) > maxLength:
pass
query = QueryParser(Version.LUCENE_CURRENT, "contentKeyword",
analyzer).parse(search_content)
scoreDocs = searcher.search(query, 50).scoreDocs
scorer = QueryScorer(query)
formatter = SimpleHTMLFormatter("<span class=\"highlight\">", "</span>")
highlighter = Highlighter(formatter, scorer)
fragmenter = SimpleFragmenter(50)
highlighter.setTextFragmenter(fragmenter)
ret['NumOfDocs'] = str(len(scoreDocs)) + "total matching documents."
print ret['NumOfDocs']
conn = pymysql.connect(host='localhost',
user=user,
password=password,
db=db_name,
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
rst = ''
ret['search_list'] = []
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
_id = str(doc.get("id"))
print _id
sql = 'select * from webpage where id=%s'
with conn.cursor() as cursor:
cursor.execute(sql, (_id))
rst = cursor.fetchone()
titleStream = ChineseAnalyzer(Version.LUCENE_CURRENT).tokenStream(
"title", StringReader(rst['title']))
titleFragment = highlighter.getBestFragment(titleStream, rst['title'])
if titleFragment is None:
titleFragment = rst['title']
contentStream = ChineseAnalyzer(Version.LUCENE_CURRENT).tokenStream(
"content", StringReader(rst['content']))
contentFragment = highlighter.getBestFragments(contentStream,
rst['content'], 5,
'...')
ret['search_list'].append({
'title': titleFragment,
'url': rst['url'],
'content': contentFragment
})
#searcher.close()
conn.close()
return render(request, 'tjut/result.html', {
'search_list': ret['search_list'],
'search_content': search_content
})
# Get the analyzer
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
# Constructs a query parser.
queryParser = QueryParser(Version.LUCENE_CURRENT, FIELD_CONTENTS, analyzer)
# Create a query
query = queryParser.parse(QUERY_STRING)
topDocs = searcher.search(query, 50)
# Get top hits
scoreDocs = topDocs.scoreDocs
print "%s total matching documents." % len(scoreDocs)
HighlightFormatter = SimpleHTMLFormatter()
query_score = QueryScorer (query)
highlighter = Highlighter(HighlightFormatter, query_score)
# Set the fragment size. We break text in to fragment of 64 characters
fragmenter = SimpleSpanFragmenter(query_score, 64)
highlighter.setTextFragmenter(fragmenter)
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
text = doc.get(FIELD_CONTENTS)
ts = analyzer.tokenStream(FIELD_CONTENTS, StringReader(text))
print doc.get(FIELD_PATH)
print highlighter.getBestFragments(ts, text, 3, "...")
print ""