def main(cls, argv):
query = TermQuery(Term("f", "ipsum"))
scorer = QueryScorer(query)
formatter = SimpleHTMLFormatter("<span class=\"highlight\">",
"</span>")
highlighter = Highlighter(formatter, scorer)
fragmenter = SimpleFragmenter(50)
highlighter.setTextFragmenter(fragmenter)
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
tokenStream = analyzer.tokenStream("f", StringReader(cls.text))
result = highlighter.getBestFragments(tokenStream, cls.text, 5, "...")
stdout.write("<html>")
stdout.write("<style>\n")
stdout.write(".highlight {\n")
stdout.write(" background: yellow\n")
stdout.write("}\n")
stdout.write("</style>")
stdout.write("<body>")
stdout.write(result)
stdout.write("</body></html>\n")
stdout.flush()
def search_image(command):
if command == ' ':
return []
Docs = []
vm_env = getVMEnv()
vm_env.attachCurrentThread()
STORE_DIR = "index_img"
directory = SimpleFSDirectory(File(STORE_DIR))
searcher = IndexSearcher(directory, True)
analyzer = WhitespaceAnalyzer(Version.LUCENE_CURRENT)
command_dict = {}
allowed_opt = ['site']
opt = 'contents'
for i in command.split(' '):
if ':' in i:
opt, value = i.split(':')[:2]
opt = opt.lower()
if opt in allowed_opt and value != '':
command_dict[opt] = command_dict.get(opt, '') + ' ' + value
else:
seg_list = jieba.cut(i)
command_dict[opt] = command_dict.get(opt,
'') + ' ' + " ".join(seg_list)
querys = BooleanQuery()
for k, v in command_dict.iteritems():
if k == 'site':
t = Term(k, '*' + v)
query = WildcardQuery(t)
else:
query = QueryParser(Version.LUCENE_CURRENT, k, analyzer).parse(v)
querys.add(query, BooleanClause.Occur.MUST)
scoreDocs = searcher.search(querys, 10000).scoreDocs
formatter = SimpleHTMLFormatter("<font color=#FF0000>", "</font>")
highlighter = Highlighter(formatter, QueryScorer(querys))
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
doc_dic = {}
doc_dic["url"] = doc.get("url")
doc_dic["imgurl"] = doc.get("imgurl")
doc_dic["urltitle"] = doc.get("urltitle")
text = doc.get("contents")
ts = analyzer.tokenStream(doc.get("contents"), StringReader(text))
doc_dic["contents"] = highlighter.getBestFragments(ts, text, 2, "...")
Docs.append(doc_dic)
searcher.close()
return Docs
def get(self, searchterm):
#Lucene Search for reference/word
query = QueryParser(Version.LUCENE_35,"text",self.analyzer).parse(searchterm)
hits = self.searcher.search(query,100000)
query_score = QueryScorer(query)
print query_score
HighlightFormatter = SimpleHTMLFormatter();
query_score = QueryScorer (query)
highlighter = Highlighter(HighlightFormatter, query_score)
# Set the fragment size. We break text in to fragment of 64 characters
fragmenter = SimpleSpanFragmenter(query_score, 64);
highlighter.setTextFragmenter(fragmenter);
# add a highlighter
wordhits = []
for hit in hits.scoreDocs:
wordhit = {}
doc = self.searcher.doc(hit.doc)
text = doc.get("text")
ts = self.analyzer.tokenStream("text", StringReader(text))
wordhit['reference'] = doc.get('reference')
wordhit['text'] = highlighter.getBestFragments(ts,text,3)
wordhit['score'] = hit.score
wordhits.append(wordhit)
# postgresql search for word details
cursor = yield gen.Task(self.db.execute, "select * from worddetailsbyword(%s);", (searchterm,))
worddetails = cursor.fetchall()
# postgresql search for person hits
cursor = yield gen.Task(self.db.execute, "select * from getrelatedbyname(%s);", (searchterm.capitalize(),))
p_res = cursor.fetchall()
# parse this stuff out before sending to template so that the logic stays here
people = []
if len(p_res) > 0:
person = {}
for p in p_res:
if p[4] == 0:
if len(person.keys()) is not 0:
people.append(person)
person = { 'name': p[1], 'surname': p[2], 'information': p[3], 'relationships': [] }
else:
person['relationships'].append({'name':p[1],'relationshiptype':p[5]})
people.append(person)
self.render("search.html",worddetails=worddetails,wordhits=wordhits,people=people)
def main(cls, argv):
query = TermQuery(Term("f", "ipsum"))
scorer = QueryScorer(query)
formatter = SimpleHTMLFormatter("<span class=\"highlight\">", "</span>")
highlighter = Highlighter(formatter, scorer)
fragmenter = SimpleFragmenter(50)
highlighter.setTextFragmenter(fragmenter)
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
tokenStream = analyzer.tokenStream("f", StringReader(cls.text))
result = highlighter.getBestFragments(tokenStream, cls.text, 5, "...")
stdout.write("<html>")
stdout.write("<style>\n")
stdout.write(".highlight {\n")
stdout.write(" background: yellow\n")
stdout.write("}\n")
stdout.write("</style>")
stdout.write("<body>")
stdout.write(result)
stdout.write("</body></html>\n")
stdout.flush()
# Get the analyzer
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
# Constructs a query parser.
queryParser = QueryParser(Version.LUCENE_CURRENT, FIELD_CONTENTS, analyzer)
# Create a query
query = queryParser.parse(QUERY_STRING)
topDocs = searcher.search(query, 50)
# Get top hits
scoreDocs = topDocs.scoreDocs
print "%s total matching documents." % len(scoreDocs)
HighlightFormatter = SimpleHTMLFormatter()
query_score = QueryScorer (query)
highlighter = Highlighter(HighlightFormatter, query_score)
# Set the fragment size. We break text in to fragment of 64 characters
fragmenter = SimpleSpanFragmenter(query_score, 64)
highlighter.setTextFragmenter(fragmenter)
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
text = doc.get(FIELD_CONTENTS)
ts = analyzer.tokenStream(FIELD_CONTENTS, StringReader(text))
print doc.get(FIELD_PATH)
print highlighter.getBestFragments(ts, text, 3, "...")
print ""
def search(request):
vm_env = lucene.getVMEnv()
vm_env.attachCurrentThread()
ret = {}
maxLength = 38
search_content = request.GET.get('content')
if len(search_content) > maxLength:
pass
query = QueryParser(Version.LUCENE_CURRENT, "contentKeyword",
analyzer).parse(search_content)
scoreDocs = searcher.search(query, 50).scoreDocs
scorer = QueryScorer(query)
formatter = SimpleHTMLFormatter("<span class=\"highlight\">", "</span>")
highlighter = Highlighter(formatter, scorer)
fragmenter = SimpleFragmenter(50)
highlighter.setTextFragmenter(fragmenter)
ret['NumOfDocs'] = str(len(scoreDocs)) + "total matching documents."
print ret['NumOfDocs']
conn = pymysql.connect(host='localhost',
user=user,
password=password,
db=db_name,
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
rst = ''
ret['search_list'] = []
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
_id = str(doc.get("id"))
print _id
sql = 'select * from webpage where id=%s'
with conn.cursor() as cursor:
cursor.execute(sql, (_id))
rst = cursor.fetchone()
titleStream = ChineseAnalyzer(Version.LUCENE_CURRENT).tokenStream(
"title", StringReader(rst['title']))
titleFragment = highlighter.getBestFragment(titleStream, rst['title'])
if titleFragment is None:
titleFragment = rst['title']
contentStream = ChineseAnalyzer(Version.LUCENE_CURRENT).tokenStream(
"content", StringReader(rst['content']))
contentFragment = highlighter.getBestFragments(contentStream,
rst['content'], 5,
'...')
ret['search_list'].append({
'title': titleFragment,
'url': rst['url'],
'content': contentFragment
})
#searcher.close()
conn.close()
return render(request, 'tjut/result.html', {
'search_list': ret['search_list'],
'search_content': search_content
})
