def displayResults(self, query, sort): searcher = IndexSearcher(self.directory, True) fillFields = False computeMaxScore = False docsScoredInOrder = False computeScores = True collector = TopFieldCollector.create(sort, 20, fillFields, computeScores, computeMaxScore, docsScoredInOrder) searcher.search(query, None, collector) scoreDocs = collector.topDocs().scoreDocs print "\nResults for:", query, "sorted by", sort print "Title".rjust(30), "pubmonth".rjust(10), \ "id".center(4), "score".center(15) for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) title = doc["title"] if len(title) > 30: title = title[:30] print title.encode('ascii', 'replace').rjust(30), \ doc["pubmonth"].rjust(10), \ str(scoreDoc.doc).center(4), \ ("%06f" % (scoreDoc.score)).rjust(12) print " ", doc["category"] # print searcher.explain(query, scoreDoc.doc) searcher.close()
def search(r, keyword=""): import logging logger = logging.getLogger("search") bench = Benchmark(logger) from lucene import IndexSearcher, StandardAnalyzer, FSDirectory, QueryParser, File, Hit import lucene, os os.environ["JAVA_HOME"] = "/usr/local/jdk1.6.0_17" lucene.initVM(lucene.CLASSPATH) directory = FSDirectory.open(File(CONFIG.INDEX_PATH)) ROBOT_INDEX = IndexSearcher(directory, True) ROBOT_ANALYZER = StandardAnalyzer() keyword = keyword or r.GET["keyword"] query = QueryParser("context", ROBOT_ANALYZER) query = query.parse('"%s"' % keyword) bench.start_mark("search") hits = ROBOT_INDEX.search(query) count = len(hits) result = [] i = 0 for hit in hits: i += 1 if i > 100: break doc = Hit.cast_(hit).getDocument() result.append(SearchResult(doc, i, keyword)) ROBOT_INDEX.close() et = bench.stop_mark() return render_to_response("robot_search_result.html", {"result": result, "count": count, "elaspe": et})
def testTermRangeQuery(self): searcher = IndexSearcher(self.directory, True) query = TermRangeQuery("title2", "d", "j", True, True) topDocs = searcher.search(query, 100) self.assertEqual(3, topDocs.totalHits) searcher.close()
def testTermRangeQuery(self): searcher = IndexSearcher(self.directory, True) query = TermRangeQuery("title2", "d", "j", True, True) topDocs = searcher.search(query, 100) self.assertEqual(3, topDocs.totalHits) searcher.close()
def main(cls, argv): if len(argv) != 2: print "Usage: BerkeleyDbSearcher <index dir>" return dbHome = argv[1] env = DBEnv() env.set_flags(DB_LOG_INMEMORY, 1); if os.name == 'nt': env.set_cachesize(0, 0x4000000, 1) elif os.name == 'posix': from commands import getstatusoutput if getstatusoutput('uname') == (0, 'Linux'): env.set_cachesize(0, 0x4000000, 1) env.open(dbHome, (DB_THREAD | DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_TXN), 0) index = DB(env) blocks = DB(env) txn = None try: txn = env.txn_begin(None) index.open(filename = '__index__', dbtype = DB_BTREE, flags = DB_THREAD, txn = txn) blocks.open(filename = '__blocks__', dbtype = DB_BTREE, flags = DB_THREAD, txn = txn) except: if txn is not None: txn.abort() txn = None raise else: txn.commit() txn = None try: txn = env.txn_begin(None) directory = DbDirectory(txn, index, blocks, 0) searcher = IndexSearcher(directory, True) topDocs = searcher.search(TermQuery(Term("contents", "fox")), 50) print topDocs.totalHits, "document(s) found" searcher.close() except: if txn is not None: txn.abort() txn = None raise else: txn.abort() index.close() blocks.close() env.close()
def getHitCount(self, fieldName, searchString): searcher = IndexSearcher(self.dir, True) t = Term(fieldName, searchString) query = TermQuery(t) hitCount = len(searcher.search(query, 50).scoreDocs) searcher.close() return hitCount
def getHitCount(self, fieldName, searchString): searcher = IndexSearcher(self.dir, True) t = Term(fieldName, searchString) query = TermQuery(t) hitCount = len(searcher.search(query, 50).scoreDocs) searcher.close() return hitCount
def SearchFiles(command): STORE_DIR = "lucene/index" getVMEnv().attachCurrentThread() # print 'lucene', VERSION directory = SimpleFSDirectory(File(STORE_DIR)) searcher = IndexSearcher(directory, True) analyzer = StandardAnalyzer(Version.LUCENE_CURRENT) rankedfiles = run(searcher, analyzer, command) searcher.close() return rankedfiles
def search(command): STORE_DIR = "index" vm_env = initVM() print 'lucene', VERSION directory = SimpleFSDirectory(File(STORE_DIR)) searcher = IndexSearcher(directory, True) analyzer = StandardAnalyzer(Version.LUCENE_CURRENT) result = run(searcher, analyzer, command) searcher.close() return result
def Searchfile(command, prior, page, RPP): STORE_DIR = "index_ans" directory = SimpleFSDirectory(File(STORE_DIR)) searcher = IndexSearcher(directory, True) analyzer = WhitespaceAnalyzer(Version.LUCENE_CURRENT) store = run(searcher, analyzer, command, prior) searcher.close() start = (page - 1) * RPP end = start + RPP return store[start:end], len(store)
def testExclusive(self): searcher = IndexSearcher(self.directory, True) # pub date of TTC was October 1988 query = NumericRangeQuery.newIntRange("pubmonth", 198805, 198810, False, False) topDocs = searcher.search(query, 100) self.assertEqual(0, topDocs.totalHits) searcher.close()
def GET(self): form1 = login() user_data = web.input() vm_env.attachCurrentThread() STORE_DIR = "F:\\imgindex" directory = SimpleFSDirectory(File(STORE_DIR)) searcher = IndexSearcher(directory, True) analyzer = SimpleAnalyzer(Version.LUCENE_CURRENT) a,b,c,d,e = img_func(user_data.keyword,searcher,analyzer) searcher.close() return render.img_result(form1,a,b,c,d,e)
class LuceneSearch(object): def __init__(self): STORE_DIR = "index" initVM() print 'lucene', VERSION self.directory = SimpleFSDirectory(File(STORE_DIR)) print self.directory self.searcher = IndexSearcher(self.directory, True) self.analyzer = StandardAnalyzer(Version.LUCENE_CURRENT) def close(self): self.searcher.close() def raw_search(self, query_string): query = QueryParser(Version.LUCENE_CURRENT, "contents", self.analyzer).parse(query_string) scoreDocs = self.searcher.search(query, 50).scoreDocs print "%s total matching documents." % len(scoreDocs) matches = [] for scoreDoc in scoreDocs: doc = self.searcher.doc(scoreDoc.doc) #print 'doc matched = ', dir(doc) contents = LuceneDoc.load(doc.get('name')) matches.append({'contents' : contents, 'doc' : doc}) return matches def search(self, query): matches = self.raw_search(query) results = '' if len(matches) > 0: results += str(len(matches))+" results <br/>" for match in matches: results += '<a href='+str(match['contents']['dealUrl'])+'>'+str(match['contents']['merchant'])+'</a><br />' results += '<p>'+str(match['contents']['shortAnnouncementTitle'])+','+str(match['contents']['redemptionLocation'])+'</p><br/>' else: results = "0 results <br/>" return results def cli_search(self): while True: print print "Hit enter with no input to quit." command = raw_input("Query:") if command == '': return matches = self.raw_search(command) print print "Searching for:", command for match in matches: print match['contents']['dealUrl'] print match['contents']['merchant'], ',', match['contents']['redemptionLocation'], ', ', match['contents']['div'] print match['contents']['shortAnnouncementTitle'] print '-'*80
def testExclusive(self): searcher = IndexSearcher(self.directory, True) # pub date of TTC was October 1988 query = NumericRangeQuery.newIntRange("pubmonth", Integer(198805), Integer(198810), False, False) topDocs = searcher.search(query, 100) self.assertEqual(0, topDocs.totalHits) searcher.close()
def begining(command): STORE_DIR = "index" global vm_env vm_env = initVM() vm_env.attachCurrentThread() #print 'lucene', VERSION directory = SimpleFSDirectory(File(STORE_DIR)) searcher = IndexSearcher(directory, True) analyzer = lucene.WhitespaceAnalyzer(Version.LUCENE_CURRENT) a = run(command, searcher, analyzer) searcher.close() return a
def testTerm(self): searcher = IndexSearcher(self.directory, True) t = Term("subject", "ant") query = TermQuery(t) scoreDocs = searcher.search(query, 50).scoreDocs self.assertEqual(1, len(scoreDocs), "JDwA") t = Term("subject", "junit") scoreDocs = searcher.search(TermQuery(t), 50).scoreDocs self.assertEqual(2, len(scoreDocs)) searcher.close()
def testTerm(self): searcher = IndexSearcher(self.directory, True) t = Term("subject", "ant") query = TermQuery(t) scoreDocs = searcher.search(query, 50).scoreDocs self.assertEqual(1, len(scoreDocs), "JDwA") t = Term("subject", "junit") scoreDocs = searcher.search(TermQuery(t), 50).scoreDocs self.assertEqual(2, len(scoreDocs)) searcher.close()
def search_lucene_index(search_params, index_dir, index_metadata, records_per_page): """ Uses the query term provided to search the disease ontology lucene index """ results = [] index_dir = SimpleFSDirectory(File(index_dir)) analyzer = build_perfield_analyzer(index_metadata) searcher = IndexSearcher(index_dir) index_fields = index_metadata.keys() # Since we are paging results we want to grab what page we are on page = (int(search_params.get('page', 1))) - 1 # Doing something pretty hacky here since we are trying to move from 0-based to 1 # based indexing to match our pagingation display offset = int(page) * records_per_page # If we are executing an advanced search we will be building a BooleanQuery # in parts as opposed to the one MultiFieldQueryParser when doing a basic # serach query = None if search_params.get('adv_search') == "True": query = build_advanced_search_query(search_params, search_params.get('operator'), analyzer) else: parser = MultiFieldQueryParser(Version.LUCENE_CURRENT, index_fields, analyzer) query = MultiFieldQueryParser.parse( parser, process_query_param((search_params.get('q')))) # Parse through our hits hits = searcher.search(query, 10000) total_hits = hits.totalHits count = min(hits.totalHits - offset, records_per_page) for i in xrange(0, count): score_doc = hits.scoreDocs[offset + i] doc = searcher.doc(score_doc.doc) term_id = doc.get('term id') name = doc.get('name') explain = searcher.explain(query, score_doc.doc) match_fields = get_field_matches(explain.toString(), index_fields) results.append((term_id, name, list(match_fields))) searcher.close() return (results, total_hits)
def search_image(command): if command == ' ': return [] Docs = [] vm_env = getVMEnv() vm_env.attachCurrentThread() STORE_DIR = "index_img" directory = SimpleFSDirectory(File(STORE_DIR)) searcher = IndexSearcher(directory, True) analyzer = WhitespaceAnalyzer(Version.LUCENE_CURRENT) command_dict = {} allowed_opt = ['site'] opt = 'contents' for i in command.split(' '): if ':' in i: opt, value = i.split(':')[:2] opt = opt.lower() if opt in allowed_opt and value != '': command_dict[opt] = command_dict.get(opt, '') + ' ' + value else: seg_list = jieba.cut(i) command_dict[opt] = command_dict.get(opt, '') + ' ' + " ".join(seg_list) querys = BooleanQuery() for k, v in command_dict.iteritems(): if k == 'site': t = Term(k, '*' + v) query = WildcardQuery(t) else: query = QueryParser(Version.LUCENE_CURRENT, k, analyzer).parse(v) querys.add(query, BooleanClause.Occur.MUST) scoreDocs = searcher.search(querys, 10000).scoreDocs formatter = SimpleHTMLFormatter("<font color=#FF0000>", "</font>") highlighter = Highlighter(formatter, QueryScorer(querys)) for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) doc_dic = {} doc_dic["url"] = doc.get("url") doc_dic["imgurl"] = doc.get("imgurl") doc_dic["urltitle"] = doc.get("urltitle") text = doc.get("contents") ts = analyzer.tokenStream(doc.get("contents"), StringReader(text)) doc_dic["contents"] = highlighter.getBestFragments(ts, text, 2, "...") Docs.append(doc_dic) searcher.close() return Docs
def _create_pairs(self, inpDF, colname, idd): """ function to create cartesian pairs of matched-similar text records first calls create index function followed by search index row by row in a pandas dataframe """ lookup_dir = {} for i, row in inpDF.iterrows(): if row[colname] not in lookup_dir: lookup_dir[row[colname]] = [] else: lookup_dir[row[colname]].append(row[idd]) pairs = [] directory = self._createIndex(inpDF, colname) searcher = IndexSearcher(directory, True) matches = inpDF.apply( lambda x: self._searchIndex(searcher, x, colname, idd), axis=1) captured_candidates = {} for match_pair in matches: for matched in match_pair: # value_index = inpDF[inpDF[colname] == matched[2]].index.tolist() value_index = lookup_dir[matched[2]] for cell_index in value_index: if matched[0] != cell_index: rstring = "-".join( sorted([str(matched[0]), str(cell_index)])) if rstring not in captured_candidates: captured_candidates[rstring] = 1 row = [] row.append(matched[0]) row.append(matched[1]) row.append(cell_index) row.append(matched[2]) pairs.append(row) searcher.close() directory.close() header = [idd, colname, idd + "_", colname + "_"] pairDF = pd.DataFrame(pairs, columns=header) return pairDF
def run(command): if command == '': return None STORE_DIR = "index" initVM(CLASSPATH) directory = FSDirectory.getDirectory(STORE_DIR, False) searcher = IndexSearcher(directory) analyzer = StandardAnalyzer() parser = QueryParser("contents", analyzer) parser.setDefaultOperator(QueryParser.Operator.AND) parser.setFuzzyMinSim(0.2) query = parser.parse(command) hits = map(transform, searcher.search(query)) searcher.close() return hits
def testCollecting(self): query = TermQuery(Term("contents", "junit")) searcher = IndexSearcher(self.directory, True) collector = BookLinkCollector(searcher) searcher.search(query, collector) links = collector.getLinks() self.assertEqual("java development with ant", links["http://www.manning.com/antbook"]) scoreDocs = searcher.search(query, 10).scoreDocs self.dumpHits(searcher, scoreDocs) searcher.close()
def search_lucene_index(search_params, index_dir, index_metadata, records_per_page): """ Uses the query term provided to search the disease ontology lucene index """ results = [] index_dir = SimpleFSDirectory(File(index_dir)) analyzer = build_perfield_analyzer(index_metadata) searcher = IndexSearcher(index_dir) index_fields = index_metadata.keys() # Since we are paging results we want to grab what page we are on page = (int(search_params.get('page', 1))) - 1 # Doing something pretty hacky here since we are trying to move from 0-based to 1 # based indexing to match our pagingation display offset = int(page) * records_per_page # If we are executing an advanced search we will be building a BooleanQuery # in parts as opposed to the one MultiFieldQueryParser when doing a basic # serach query = None if search_params.get('adv_search') == "True": query = build_advanced_search_query(search_params, search_params.get('operator'), analyzer) else: parser = MultiFieldQueryParser(Version.LUCENE_CURRENT, index_fields, analyzer) query = MultiFieldQueryParser.parse(parser, process_query_param((search_params.get('q')))) # Parse through our hits hits = searcher.search(query, 10000) total_hits = hits.totalHits count = min(hits.totalHits - offset, records_per_page) for i in xrange(0, count): score_doc = hits.scoreDocs[offset+i] doc = searcher.doc(score_doc.doc) term_id = doc.get('term id') name = doc.get('name') explain = searcher.explain(query, score_doc.doc) match_fields = get_field_matches( explain.toString(), index_fields ) results.append( (term_id, name, list(match_fields)) ) searcher.close() return (results, total_hits)
def doSearch(self,searchString,fieldToSearch,luceneDir): searchResult =[] store = SimpleFSDirectory(File(luceneDir)) analyzer = StandardAnalyzer(Version.LUCENE_CURRENT) searcher = IndexSearcher(store) query = QueryParser(Version.LUCENE_CURRENT,fieldToSearch,analyzer).parse(searchString) hits = searcher.search(query,self.MAX) print "Found %d documents that matched the query '%s'" %(hits.totalHits,searchString) for hit in hits.scoreDocs: doc = searcher.doc(hit.doc) #docdict['score'] = hit.score #docdict['docid'] = hit.doc #docdict['content'] = doc.get("contents").encode("utf-8") searchResult.append([doc.get("title").encode("utf-8"),doc.get("contents").encode("utf-8")]) searcher.close() return searchResult
def delete_old(self, index): existing_ids = set([book.id for book in Book.objects.all()]) reader = IndexReader.open(index.index, False) searcher = IndexSearcher(reader) try: num = searcher.docFreq(Term('is_book', 'true')) docs = searcher.search(Search.make_term_query(['true'], 'is_book'), num) for result in docs.scoreDocs: stored = searcher.doc(result.doc) book_id = int(stored.get('book_id')) if not book_id in existing_ids: print "book id %d doesn't exist." % book_id index.remove_book(book_id) finally: searcher.close() reader.close()
def testSimple(self): class SimpleSimilarity(PythonSimilarity): def lengthNorm(_self, field, numTerms): return 1.0 def queryNorm(_self, sumOfSquaredWeights): return 1.0 def tf(_self, freq): return freq def sloppyFreq(_self, distance): return 2.0 def idfTerms(_self, terms, searcher): return 1.0 def idf(_self, docFreq, numDocs): return 1.0 def coord(_self, overlap, maxOverlap): return 1.0 def scorePayload(_self, docId, fieldName, start, end, payload, offset, length): return 1.0 self.indexSingleFieldDocs([Field("contents", "x", Field.Store.YES, Field.Index.ANALYZED)]) searcher = IndexSearcher(self.directory) searcher.setSimilarity(SimpleSimilarity()) query = TermQuery(Term("contents", "x")) explanation = searcher.explain(query, 0) print explanation scoreDocs = searcher.search(query, 50).scoreDocs self.assertEqual(1, len(scoreDocs)) self.assertEqual(scoreDocs[0].score, 1.0) searcher.close()
def a(): import os from lucene import CJKAnalyzer,Hit dire = os.path.dirname(__file__) + '/index' analyzer = CJKAnalyzer() searcher = IndexSearcher(dire) query = QueryParser('summary',analyzer).parse('java')#TermQuery(Term("type", "restaurant")) sort = Sort(SortField("locatisdon", CategoryComparatorSource('java'))) hits = searcher.search(query,sort) print len(hits) i = 0 for hit in hits: i+=1 if i== 10: break doc = Hit.cast_(hit).getDocument() print 'title:', doc.get("author"), 'name:', doc.get("link") print Hit.cast_(hit).getScore() searcher.close()
def calculateWeight(self,context): #try: self.termList = context.termList ramreader = IndexReader.open(context.ramIndex,True) store = SimpleFSDirectory(File(context.luceneDir)) storereader = IndexReader.open(store) searcher = IndexSearcher(store) ramsearcher = IndexSearcher(context.ramIndex) # Number of documents in the collection N = storereader.numDocs() # Number of relevant documents R = ramreader.numDocs() analyzer = SimpleAnalyzer(Version.LUCENE_CURRENT) for w in self.termList: searchString= "'" + w + "'" query = QueryParser(Version.LUCENE_CURRENT,"contents",analyzer).parse(searchString) # Number of relevant document having the term #r = ramsearcher.docFreq(Term("contents",w)) hits = ramsearcher.search(query,self.MAX) r = hits.totalHits # Number of documents having the term #n = searcher.docFreq(Term("contents",w)) query = QueryParser(Version.LUCENE_CURRENT,context.searchField,analyzer).parse(searchString) hits = searcher.search(query,self.MAX) n = hits.totalHits if (R-r) > 0 and (n-r) > 0 and (N-n-R+r) > 0: weight = (r/(R-r))/(((n-r)/(N-n-R+r))) else: weight =0 if weight > self.weightThreshold: self.gramList.append([w,weight]) searcher.close() ramsearcher.close() storereader.close() ramreader.close() #except Exception,e: # print 'error',e
def similar(command, docno): STORE_DIR = "index" initVM(CLASSPATH) directory = FSDirectory.getDirectory(STORE_DIR, False) searcher = IndexSearcher(directory) analyzer = StandardAnalyzer() parser = QueryParser("contents", analyzer) parser.setDefaultOperator(QueryParser.Operator.AND) parser.setFuzzyMinSim(0.2) query = parser.parse(command) hits = searcher.search(query) document = hits.id(docno) ir = IndexReader.open(STORE_DIR) mlt = MoreLikeThis(ir) mlt.setFieldNames(['name', 'contents']) mlt.setMinWordLen(2) mlt.setBoost(True) query = mlt.like(document) hits = map(transform, searcher.search(query)) searcher.close() return hits
class Searcher(object): def __init__(self): self.searcher = IndexSearcher(STORE_DIR) self.analyzer = CJKAnalyzer() self.catfilter = CatFilter() def __del__(self): self.searcher.close() def search(self, query,category_id=None): SHOULD = BooleanClause.Occur.SHOULD #MultiFieldQueryParser.setOperator(QueryParser.DEFAULT_OPERATOR_AND); parser1 = QueryParser('summary',self.analyzer) parser2 = QueryParser('title',self.analyzer) parser1.setDefaultOperator(QueryParser.AND_OPERATOR) parser2.setDefaultOperator(QueryParser.AND_OPERATOR) q1 = parser1.parse(query) q2 = parser2.parse(query) boolQuery = BooleanQuery() boolQuery.add(q1,SHOULD) boolQuery.add(q2,SHOULD) #camp = CategoryComparatorSource(query) #sortfield = SortField("link", camp) #sort = Sort(sortfield) if category_id: self.catfilter.query = query self.catfilter.category_id = category_id hits = self.searcher.search(boolQuery,self.catfilter) else: hits = self.searcher.search(boolQuery) return hits def search_by_field(self,query,field='summary'): parser = QueryParser(field,self.analyzer) parser.setDefaultOperator(QueryParser.AND_OPERATOR) q = parser.parse(query) return self.searcher.search(q)
class tfidf(Command): """ """ def __init__(self): """ """ # Number of documents self.N = 0 # Number of terms self.m = 0 # Individual terms self.unigramList = None def process(self,context): self.unigramList = context.termList self.ramreader = IndexReader.open(context.ramIndex,True) self.ramsearcher = IndexSearcher(context.ramIndex) self.N = self.ramreader.numDocs() self.m = len(self.unigramList) self.createTermDocumentMatrix() self.ramsearcher.close() self.ramreader.close() context.termdocumentmatrix = self.termdocumentMatrix print 'finished creating term document matrix' self.context = context def createTermDocumentMatrix(self): self.termdocumentMatrix = np.zeros((self.m,self.N),dtype=int) analyzer = SimpleAnalyzer(Version.LUCENE_CURRENT) for index,word in enumerate(self.unigramList): searchString= "'" + word[0] + "'" query = QueryParser(Version.LUCENE_CURRENT,"contents",analyzer).parse(searchString) hits = self.ramsearcher.search(query,self.N) for hit in hits.scoreDocs: self.termdocumentMatrix[index,hit.doc] = hits.totalHits
def retrieve(string,tweetID): global eventNum global eventDict global eventList lucene.initVM() indexDir = "REMOVEME.index-dir" dir = SimpleFSDirectory(File(indexDir)) analyzer = StandardAnalyzer(Version.LUCENE_30) try: searcher = IndexSearcher(dir) except lucene.JavaError: #print 'Inside First Except' eventDict[tweetID] = eventNum eventNum = eventNum + 1 analyzer.close() return try: query = QueryParser(Version.LUCENE_30, "text", analyzer).parse(string) #e = sys.exc_info()[0] #print e MAX = 2 hits = searcher.search(query, MAX) #print "Found %d document(s) that matched query '%s':" % (hits.totalHits, query) #print 'total hits' #print hits.totalHits if hits.totalHits > 0: eventDict[tweetID] = eventDict[hits.scoreDocs[0].doc] analyzer.close() searcher.close() return else: #print '-----------' #print tweetID eventDict[tweetID] = eventNum eventNum = eventNum + 1 analyzer.close() searcher.close() return #for hit in hits.scoreDocs: #print hit.score, hit.doc, hit.toString() #doc = searcher.doc(hit.doc) #print doc.get("text").encode("utf-8") except lucene.JavaError: eventDict[tweetID] = eventNum eventNum = eventNum + 1 analyzer.close() searcher.close() return
def do_invite(keywords): print "invite started!>>>>>>" initVM() indexDir = "/tmp/luceneindex/doc" version = Version.LUCENE_CURRENT idxDir = SimpleFSDirectory(File(indexDir)) analyzer = StandardAnalyzer(version) searcher = IndexSearcher(idxDir) query = QueryParser(version, "description", analyzer).parse(keywords) hits = searcher.search(query, 1000) indentCandidates = [] #print len(hits.scoreDocs) for hit in hits.scoreDocs: print hit.score, hit.doc, hit.toString() doc = searcher.doc(hit.doc) print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>" #print doc.get("description") intents = doc.get("intent") #print doc.get("url") if intents == None: continue intents = eval(intents) for intent in intents: indentCandidates.append(intent) searcher.close() inviteEmails = [] #patterns = ["[^A-Za-z0-9_-]*(?P<buzz>([A-Za-z0-9_-]+(\.\w+)*@(\w+\.)+\w{2,3}))", '''qq[^\d]*(?P<buzz>[1-9][0-9]{4,})'''] for indentCandidate in indentCandidates: #print repr(indentCandidate[0]) emailCandidate = indentCandidate[0] if emailCandidate.find("@") == -1: qqMail = emailCandidate+"@qq.com" inviteEmails.append(qqMail) else: inviteEmails.append(emailCandidate) # remove useless intent #indentCandidate = indentCandidate.strip() #probability = bayes.checkneedprobability((indentCandidate).decode("ascii","ignore").encode("utf8")) #print probability #if (probability>0.5): # continue #emailPattern = patterns[0] #qqPattern = patterns[1] #qqMatches =re.finditer(qqPattern,indentCandidate, re.IGNORECASE| re.DOTALL) #emailMatches = re.finditer(emailPattern,indentCandidate, re.IGNORECASE| re.DOTALL) #for qqMatch in qqMatches: # qq = qqMatch.group("buzz").strip() #print qq # qqMail = qq+"@qq.com" # inviteEmails.append(qqMail) #for emailMatch in emailMatches: # email = emailMatch.group("buzz").strip() #print email # inviteEmails.append(email) #add haiming and rex mail #remove multipule emails toInviteEmails = set(inviteEmails) toInviteEmails.add("*****@*****.**") toInviteEmails.add("*****@*****.**") toInviteEmails.add("*****@*****.**") print "invite total" +str(len(toInviteEmails))+" emails >>>>>>>>>>>" #for mail_address in toInviteEmails: # print mail_address return toInviteEmails
print "Searching for:", command query = QueryParser(Version.LUCENE_CURRENT, "contents", analyzer).parse(command) #用analyzer来对查询语句进行词法分析和语言处理。 #QueryParser调用parser进行语法分析,形成查询语法树,放到Query中。 scoreDocs = searcher.search(query, 50).scoreDocs #IndexSearcher调用search对查询语法树Query进行搜索,得到结果 print "%s total matching documents." % len(scoreDocs), '\n' for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) print "------------------------" print 'path:', doc.get("path") print 'title:', doc.get("title") print 'url:', doc.get("url") print 'name:', doc.get("name") if __name__ == '__main__': STORE_DIR = "index" initVM() print 'lucene', VERSION directory = SimpleFSDirectory(File(STORE_DIR)) #索引文件存放的位置 searcher = IndexSearcher(directory, True) #索引信息读入到内存,创建IndexSearcher准备进行搜索 analyzer = lucene.WhitespaceAnalyzer( Version.LUCENE_CURRENT ) #analyzer用来对查询语句进行词法分析和语言处理的,和IndexFiles.py中使用同样的analyzer。 run(searcher, analyzer) searcher.close()
def queryenhancement(query, swords, k, fList, path, TotalN): if not os.path.isdir(path): print("No directory named %s" % os.path.abspath(path)) return if len(fList) == 0: return "" wordcount = {} N = len(fList) # def length of docs doclen = np.zeros(N) # def docs query docquery = np.zeros(N) # proximity to query word proxword = {} # tokenize query queryword = re.findall('[0-9a-zA-Z]+', query) for i in range(N): fname = fList[i] fp = open(path + "/" + str(fname)) # tokenize doc docword = [s.lower() for s in re.findall("[0-9a-zA-Z]+", fp.read())] # length of docs doclen[i] = len(docword) # l(D) # store the location of query words queryloc = [] for j in range(len(docword)): # occurence of words if docword[j] not in wordcount: wordcount[docword[j]] = np.zeros(N) wordcount[docword[j]][i] = 1 else: wordcount[docword[j]][i] += 1 # record token location if docword[j].lower() in queryword: docquery[i] += 1 # m(Q,D) queryloc.append(j) proxanc = [ filter(lambda y: y < 6 and y > 0, range(x - 5, x + 6)) for x in queryloc ] for ancL in proxanc: for anc in ancL: if docword[anc] not in proxword: proxword[docword[anc]] = np.zeros(N) proxword[docword[anc]][i] = 1 else: proxword[docword[anc]][i] += 1 fp.close() # exclude stopwords and query words for w in wordcount.keys(): if w in queryword or w in swords: wordcount.pop(w) # c(W,D) for w in proxword.keys(): if w in queryword or w in swords: proxword.pop(w) # align proximity measurement and word frequency for w in wordcount.keys(): if w not in proxword: proxword[w] = np.zeros(N) # f(W,Q,D) rwordocc = {w: np.sum(wordcount[w] != 0) for w in wordcount} # z(W,Q) STORE_DIR = "lucene/index" getVMEnv().attachCurrentThread() directory = SimpleFSDirectory(File(STORE_DIR)) ireader = IndexSearcher(directory, True) docfreq = { w: ireader.docFreq(Term("contents", w.lower())) for w in wordcount } # g(W) ireader.close() # label the words wordtag = np.array(wordcount.keys()) wordlabel = {x: word for x, word in enumerate(wordtag)} score1 = np.zeros([len(wordcount), N]) score2 = np.zeros([len(wordcount), N]) y = np.zeros(len(wordcount)) for i in range(len(wordcount)): w = wordlabel[i] score1[i] = proxword[w] score2[i] = wordcount[w] y[i] = np.max([0, (rwordocc[w] * 1. / N - 2. * docfreq[w] / TotalN)]) score1 *= 1. / docquery score2 *= 1. / doclen score1 = np.sqrt(score1) score2 = np.sqrt(score2) vals = np.sum(score1 + (y * score2.T).T, axis=1) idx = np.argsort(vals)[::-1] print vals[idx[:100]] print docquery ret = "" count = 0 for i in range(len(idx)): # if wordtag[idx[i]] not in query and wordtag[idx[i]] not in swords: ret += " " + wordtag[idx[i]] count += 1 if count < k: continue else: break # print ret return ret
class SynonymAnalyzerTest(TestCase): synonymAnalyzer = SynonymAnalyzer(MockSynonymEngine()) def setUp(self): self.directory = RAMDirectory() writer = IndexWriter(self.directory, self.synonymAnalyzer, True, IndexWriter.MaxFieldLength.UNLIMITED) doc = Document() doc.add( Field("content", "The quick brown fox jumps over the lazy dogs", Field.Store.YES, Field.Index.ANALYZED)) writer.addDocument(doc) writer.close() self.searcher = IndexSearcher(self.directory, True) def tearDown(self): self.searcher.close() def testJumps(self): stream = self.synonymAnalyzer.tokenStream("contents", StringReader("jumps")) term = stream.addAttribute(TermAttribute.class_) posIncr = stream.addAttribute(PositionIncrementAttribute.class_) i = 0 expected = ["jumps", "hops", "leaps"] while stream.incrementToken(): self.assertEqual(expected[i], term.term()) if i == 0: expectedPos = 1 else: expectedPos = 0 self.assertEqual(expectedPos, posIncr.getPositionIncrement()) i += 1 self.assertEqual(3, i) def testSearchByAPI(self): tq = TermQuery(Term("content", "hops")) topDocs = self.searcher.search(tq, 50) self.assertEqual(1, topDocs.totalHits) pq = PhraseQuery() pq.add(Term("content", "fox")) pq.add(Term("content", "hops")) topDocs = self.searcher.search(pq, 50) self.assertEquals(1, topDocs.totalHits) def testWithQueryParser(self): query = QueryParser(Version.LUCENE_CURRENT, "content", self.synonymAnalyzer).parse('"fox jumps"') topDocs = self.searcher.search(query, 50) # in Lucene 1.9, position increments are no longer ignored self.assertEqual(1, topDocs.totalHits, "!!!! what?!") query = QueryParser(Version.LUCENE_CURRENT, "content", StandardAnalyzer( Version.LUCENE_CURRENT)).parse('"fox jumps"') topDocs = self.searcher.search(query, 50) self.assertEqual(1, topDocs.totalHits, "*whew*") def main(cls): query = QueryParser(Version.LUCENE_CURRENT, "content", cls.synonymAnalyzer).parse('"fox jumps"') print "\"fox jumps\" parses to ", query.toString("content") print "From AnalyzerUtils.tokensFromAnalysis: " AnalyzerUtils.displayTokens(cls.synonymAnalyzer, "\"fox jumps\"") print '' main = classmethod(main)
class PlushBase: """Base class.""" def __init__(self, store_dir=None, verbose=False): self.verbose = verbose self.store_path = None self.searcher = None self.index_reader = None self.directory = None self.analyzers = {} self.initAnalyzers() self.default_analyzer_id = 'Simple' self.fields = [] self._connected = False if store_dir: self.openStore(store_dir) def __del__(self): self.closeStore() self._connected = False def initDummyStore(self, directory): """Open a dummy ramdirectory for testing.""" writer = IndexWriter(directory, SimpleAnalyzer(), True) doc = Document() doc.add(Field("name", 'dummy.txt', Field.Store.YES, Field.Index.UN_TOKENIZED)) doc.add(Field("path", '/path/to/dummy.txt', Field.Store.YES, Field.Index.UN_TOKENIZED)) doc.add(Field("path", '/path/to/another/dummy.txt', Field.Store.YES, Field.Index.UN_TOKENIZED)) doc.add(Field("contents", "foo dummy bar", Field.Store.YES, Field.Index.TOKENIZED)) writer.addDocument(doc) writer.optimize() writer.close() def openStore(self, store_dir): """Open a lucene store.""" if self._connected: self.closeStore() if store_dir == 'dummy': directory = RAMDirectory() self.initDummyStore(directory) store_path = store_dir else: store_path = os.path.abspath(store_dir) try: directory = SimpleFSDirectory(File(store_path)) #TODO , False) except JavaError: print "Error: %s Not found." % store_path return try: self.searcher = IndexSearcher(directory) except JavaError: print "Error: '%s' is not a valid lucene store." % store_path return print 'Opening store: %s' % store_path self.directory = directory self.store_path = store_path # TODO - TaxonomyReader?? self.index_reader = IndexReader.open(directory) self.fields = self.getFieldNames() self.fields.sort() self._connected = True def closeStore(self): """Close a lucene store.""" if self.searcher is not None: if self.verbose: print "Close searcher." self.searcher.close() self.directory = None self.searcher = None self.index_reader = None self.fields = [] self.store_path = None self._connected = False def maxDoc(self): """Maximum doc number.""" return self.index_reader.maxDoc() def numDocs(self): """Number of docs in the store.""" return self.index_reader.numDocs() def getFieldNames(self): """Return a unique list of field names that exist in this index.""" fields = {} terms = self.index_reader.terms() while terms.next(): fields[terms.term().field()] = True return fields.keys() # TODO #if VERSION.startswith('1.9'): # return self.index_reader.getFieldNames() #return self.index_reader.getFieldNames(IndexReader.FieldOption.ALL) def getFields(self, doc_num=None): """Return fields of a doc.""" if doc_num is None: doc_num = self.maxDoc() - 1 doc = self.index_reader.document(doc_num) return doc.fields() def getDoc(self, doc_num=None): """Return a lucene doc.""" if doc_num is None: doc_num = self.maxDoc() - 1 return self.index_reader.document(doc_num) def getFieldInfos(self, doc_num=None): """Return fields description. [(name, stored, index, token, binary, compressed), ...]""" fields = [] doc = self.getDoc(doc_num) for name in self.fields: # TODO - this form of getFields() is deprecated mfields= doc.getFields(name) if not mfields: fields.append((name, False, False, False, False, False, 'N/A')) continue for field in mfields: fields.append((field.name(), field.isStored(), field.isIndexed(), field.isTokenized(), field.isBinary(), False, field.stringValue())) #TODO #field.isCompressed(), field.stringValue())) return fields def search(self, command, field_id="contents", sort_on=None, sort_order=False, analyzer_id=None): """Do the lucene search.""" analyzer = self.getAnalyzer(analyzer_id) try: if VERSION.startswith('1.9'): query = QueryParser.parse(command, field_id, analyzer) else: query = QueryParser(field_id, analyzer).parse(command) except JavaError: print "Error: Lucene cannot parse this query." return None if sort_on: return self.searcher.search(query, Sort(sort_on, sort_order)) return self.searcher.search(query) def getTermFreqs(self, field=None, max_term=None, pattern=None): """Return a list ([(num occurence, term), ..], number of term)""" item = self.index_reader.terms() min_freq = 0 freqs = [] if max_term: limit = max_term else: limit = 1000 if pattern is not None: try: pat = re.compile(pattern) except RegexError: print "Error: '%s' is an invalid regex" % pattern return [], 0 count = 0 while(item.next()): term = item.term() if field and term.field() != field: continue count += 1 word = term.text() freq = item.docFreq() if pattern is not None and not pat.search(word): continue if len(freqs) >= limit and freq < min_freq: continue freqs.append((-1 * freq, word)) freqs.sort() if len(freqs) > limit: freqs.pop() min_freq = freqs[0][0] item.close() freqs = [(-1*freq, word) for freq, word in freqs] return freqs, count def initAnalyzers(self): """Init all analyzer.""" # TODO #self.analyzers['French'] = FrenchAnalyzer() #self.analyzers['German'] = GermanAnalyzer() self.analyzers['Keyword'] = KeywordAnalyzer(Version.LUCENE_CURRENT) self.analyzers['Simple'] = SimpleAnalyzer(Version.LUCENE_CURRENT) self.analyzers['Stop'] = StopAnalyzer(Version.LUCENE_CURRENT) self.analyzers['Standard'] = StandardAnalyzer(Version.LUCENE_CURRENT) self.analyzers['Whitespace'] = WhitespaceAnalyzer(Version.LUCENE_CURRENT) nxlucene_home = os.getenv('NXLUCENE_HOME', None) if nxlucene_home: # point to http://svn.nuxeo.org/pub/NXLucene/trunk/src/nxlucene nxlucene_home = os.path.normpath(nxlucene_home) sys.path.append(nxlucene_home) try: from analysis import analyzers_map except ImportError: print "Error: Invalid NXLUCENE_HOME %s" % nxlucene_home return for key, value in analyzers_map.items(): self.analyzers['nx' + key] = value print "NXLucene analyzers loaded." def getAnalyzer(self, analyzer_id=None): """Return an analyzer or default.""" if analyzer_id is None: analyzer_id = self.default_analyzer_id return self.analyzers.get(analyzer_id) def displayAnalyzedQuery(self, text, field_name, analyzer_id=None): """Print analyzed tokens.""" analyzer = self.getAnalyzer(analyzer_id) tokens = [token.termText() for token in analyzer.tokenStream(field_name, StringReader(text))] print " %s analyzer tokens: %s" % (analyzer_id or self.default_analyzer_id, ", ".join(tokens) )
def search2(): results0=[] results2=[] loc=[] sr='' tmp='' mark=False if request.method == 'POST': try: initVM() directory = SimpleFSDirectory(File('albumIndex')) searcher2 = IndexSearcher(directory, True) analyzer = StandardAnalyzer(Version.LUCENE_CURRENT) if "Search" in request.form.values(): sr=request.form['text'] elif "Shuffle" in request.form.values(): mark=True while len(loc)<20: tmp=random.randint(0,searcher2.maxDoc()-1) if tmp not in loc: loc+=[tmp] if mark: print 'loc=',loc ct=0 for i in loc: doc = searcher2.doc(i) songs=doc.get('albumsongs') songs=songs.split('!@#$%') urls=doc.get("albumsongURLs") urls=urls.split('!@#$%') results2+=[{'albumnum': doc.get("albumnum"),\ 'albumname':doc.get('albumname'),\ 'albumartist':doc.get('albumartist'),\ 'albumintro': doc.get("albumintro"),\ 'albumsongs':songs,\ 'albumsongURLs': urls,\ 'albumpicURL':doc.get('albumpicURL'),\ 'albumartistURL':doc.get('albumartistURL'),\ 'albumURL':doc.get('albumURL'),\ 'rank':100}] else: print request.form.values() print 'sr=',sr if sr=='': return results0,results2,"" ## for i in sr: ## tmp+=i+" " ## print tmp scoreDocs=run2(searcher2, analyzer,sr,1) #search exact album if scoreDocs!=False: doc=scoreDocs songs=doc.get('albumsongs') songs=songs.split('!@#$%') urls=doc.get("albumsongURLs") urls=urls.split('!@#$%') results2+=[{'albumnum': doc.get("albumnum"),\ 'albumname':doc.get('albumname'),\ 'albumartist':doc.get('albumartist'),\ 'albumintro': doc.get("albumintro"),\ 'albumsongs':songs,\ 'albumsongURLs': urls,\ 'albumpicURL':doc.get('albumpicURL'),\ 'albumartistURL':doc.get('albumartistURL'),\ 'albumURL':doc.get('albumURL'),\ 'rank':100}] results0=results2 else: scoreDocs=run2(searcher2, analyzer,sr,20) #search 20 albums rank=100 for scoreDoc in scoreDocs: doc = searcher2.doc(scoreDoc.doc) songs=doc.get('albumsongs') songs=songs.split('!@#$%') urls=doc.get("albumsongURLs") urls=urls.split('!@#$%') results2+=[{'albumnum': doc.get("albumnum"),\ 'albumname':doc.get('albumname'),\ 'albumartist':doc.get('albumartist'),\ 'albumintro': doc.get("albumintro"),\ 'albumsongs':songs,\ 'albumsongURLs': urls,\ 'albumpicURL':doc.get('albumpicURL'),\ 'albumartistURL':doc.get('albumartistURL'),\ 'albumURL':doc.get('albumURL'),\ 'rank':rank}] rank-=5 conn = MySQLdb.connect(host='localhost', user='******',passwd='1234',charset="utf8") # conn = MySQLdb.connect(host='localhost', user='******',passwd='ee208',charset="utf8") conn.select_db('coversearch'); cursor = conn.cursor() for i in results2: try: cursor.execute("select zan from albums where id="+i['albumnum']) zan=cursor.fetchone()[0] i['zan']=zan i['rank']+=int(zan) except: i['zan']=0 results2.sort(key=lambda x:x['rank'],reverse=True) results2=resortalbum(results2,sr) conn.commit() cursor.close() conn.close() searcher2.close() except Exception,e: print 2,e
def search(): results=[] results3=[] loc=[] sr='' tmp='' mark=False if request.method == 'POST': try: initVM() directory = SimpleFSDirectory(File("songIndex")) searcher = IndexSearcher(directory, True) directory = SimpleFSDirectory(File("artistIndex")) searcher3 = IndexSearcher(directory, True) analyzer = StandardAnalyzer(Version.LUCENE_CURRENT) if "Search" in request.form.values(): sr=request.form['text'] elif "Shuffle" in request.form.values(): mark=True while len(loc)<20: tmp=random.randint(0,searcher.maxDoc()-1) if tmp not in loc: loc+=[tmp] ## if request.form['action']=="Search": ## sr=request.form['text'] ## elif request.form['action']=="Shuffle": ## sr='1' if mark: print 'loc=',loc for i in loc: doc = searcher.doc(i) results+=[{'songname':doc.get("songname"),\ 'songurl':doc.get('songurl'),\ 'albumname':doc.get('songalbum'),\ 'songartist':doc.get('songartist'),\ 'albumurl': doc.get("songalbumURL"),\ 'picPath':doc.get('songpicURL'),\ }] else: print request.form.values() print 'sr=',sr if sr=='': return results,results3,"" for i in sr: tmp+=i+" " ## print tmp ## scoreDocs=run2(searcher2, analyzer,sr) ## if len(scoreDocs)!=0: ## doc=searcher2.doc(scoreDocs[0].doc) ## results2+=[{'albumnum:', doc.get("albumnum"),\ ## 'albumname:',doc.get('albumname'),\ ## 'albumartist:',doc.get('albumartist'),\ ## 'albumintro:', doc.get("albumintro"),\ ## 'albumsongs:',doc.get('albumsongs'),\ ## 'albumsongURLs:', doc.get("albumsongURLs"),\ ## 'albumpicURL:',doc.get('albumpicURL')}] ## else: scoreDocs=run3(searcher3,analyzer,sr) if scoreDocs == False: scoreDocs=run(searcher, analyzer,sr) for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) results+=[{'songname':doc.get("songname"),\ 'songurl':doc.get('songurl'),\ 'albumname':doc.get('songalbum'),\ 'songartist':doc.get('songartist'),\ 'albumurl': doc.get("songalbumURL"),\ 'picPath':doc.get('songpicURL')\ }] else: doc=scoreDocs singeralbums=doc.get('singeralbums') singeralbums=singeralbums.split('!@#$%') singeralbumURLs=doc.get("singeralbumURLs") singeralbumURLs=singeralbumURLs.split('!@#$%') results3+=[{'singername': doc.get("singername"),\ 'singerplace':doc.get('singerplace'),\ 'singerintro':doc.get('singerintro'),\ 'singeralbums': singeralbums,\ 'singeralbumURLs':singeralbumURLs,\ 'singerpicURL': doc.get("singerpicURL")\ }] searcher.close() except Exception,e: print 1,e
t =c_buffer(strlen*6) bSuccess = dll.ICTCLAS_ParagraphProcess(c_char_p(command),c_int(strlen),t,c_int(0),0) command=t.value.decode('gbk').encode('utf8') ##list=t.value.split() ##print ' '.join(list) dll.ICTCLAS_Exit() command=command.decode('utf8') if command == '': return print print "Searching for:", command query = QueryParser(Version.LUCENE_CURRENT, "contents", analyzer).parse(command) scoreDocs = searcher.search(query, 50).scoreDocs print "%s total matching documents." % len(scoreDocs) for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) print 'path:', doc.get("path"), 'title:', doc.get("title"), 'url:', doc.get("url"), 'name:', doc.get("name") if __name__ == '__main__': STORE_DIR = "F:\\index" initVM() print 'lucene', VERSION directory = SimpleFSDirectory(File(STORE_DIR)) searcher = IndexSearcher(directory, True) analyzer = SimpleAnalyzer(Version.LUCENE_CURRENT) run(searcher, analyzer) searcher.close()
class SynonymAnalyzerTest(TestCase): synonymAnalyzer = SynonymAnalyzer(MockSynonymEngine()) def setUp(self): self.directory = RAMDirectory() writer = IndexWriter(self.directory, self.synonymAnalyzer, True, IndexWriter.MaxFieldLength.UNLIMITED) doc = Document() doc.add(Field("content", "The quick brown fox jumps over the lazy dogs", Field.Store.YES, Field.Index.ANALYZED)) writer.addDocument(doc) writer.close() self.searcher = IndexSearcher(self.directory, True) def tearDown(self): self.searcher.close() def testJumps(self): stream = self.synonymAnalyzer.tokenStream("contents", StringReader("jumps")) term = stream.addAttribute(TermAttribute.class_) posIncr = stream.addAttribute(PositionIncrementAttribute.class_) i = 0 expected = ["jumps", "hops", "leaps"] while stream.incrementToken(): self.assertEqual(expected[i], term.term()) if i == 0: expectedPos = 1 else: expectedPos = 0 self.assertEqual(expectedPos, posIncr.getPositionIncrement()) i += 1 self.assertEqual(3, i) def testSearchByAPI(self): tq = TermQuery(Term("content", "hops")) topDocs = self.searcher.search(tq, 50) self.assertEqual(1, topDocs.totalHits) pq = PhraseQuery() pq.add(Term("content", "fox")) pq.add(Term("content", "hops")) topDocs = self.searcher.search(pq, 50) self.assertEquals(1, topDocs.totalHits) def testWithQueryParser(self): query = QueryParser(Version.LUCENE_CURRENT, "content", self.synonymAnalyzer).parse('"fox jumps"') topDocs = self.searcher.search(query, 50) # in Lucene 1.9, position increments are no longer ignored self.assertEqual(1, topDocs.totalHits, "!!!! what?!") query = QueryParser(Version.LUCENE_CURRENT, "content", StandardAnalyzer(Version.LUCENE_CURRENT)).parse('"fox jumps"') topDocs = self.searcher.search(query, 50) self.assertEqual(1, topDocs.totalHits, "*whew*") def main(cls): query = QueryParser(Version.LUCENE_CURRENT, "content", cls.synonymAnalyzer).parse('"fox jumps"') print "\"fox jumps\" parses to ", query.toString("content") print "From AnalyzerUtils.tokensFromAnalysis: " AnalyzerUtils.displayTokens(cls.synonymAnalyzer, "\"fox jumps\"") print '' main = classmethod(main)