def search_index(indexfile, querytext, top=10, qe=False, default_field="text", display_fields=["subreddit", "author", "text"]):
lucene.initVM()
lindex = SimpleFSDirectory(Paths.get(indexfile))
ireader = DirectoryReader.open(lindex)
isearcher = IndexSearcher(ireader)
analyser = StandardAnalyzer()
parser = QueryParser(default_field, analyser)
query = parser.parse(querytext)
hits = isearcher.search(query, top).scoreDocs
docIDs = [hit.doc for hit in hits]
print_results(isearcher, hits, display_fields)
if len(hits) == 0:
print("No hits!")
elif qe:
print("\n")
print("Which documents were relevant to your search need? (Enter spaced list of result numbers [1-{}], e.g. 2 4 5)".format(top))
relevantids = [docIDs[i-1] for i in [int(x) for x in input().split()]]
nonrelevantids = [id for id in docIDs if id not in relevantids]
print("\n\n")
qequerytext = queryexpansion.rocchio(ireader, querytext, relevantids, nonrelevantids)
print("Expanded search query: '{}'\n".format(qequerytext))
qequery = parser.parse(qequerytext)
qehits = isearcher.search(qequery, top).scoreDocs
print_results(isearcher, qehits, display_fields)
ireader.close()
lindex.close()
def searchWithRequestAndQuery(cls, query, indexReader, taxoReader,
indexingParams, facetRequest):
"""
Search an index with facets for given query and facet requests.
returns a List<FacetResult>
"""
# prepare searcher to search against
searcher = IndexSearcher(indexReader)
# collect matching documents into a collector
topDocsCollector = TopScoreDocCollector.create(10, True)
if not indexingParams:
indexingParams = FacetIndexingParams.DEFAULT
# Faceted search parameters indicate which facets are we interested in
facetRequests = [facetRequest,]
facetRequests = Arrays.asList(facetRequests)
# Add the facet request of interest to the search params:
facetSearchParams = FacetSearchParams(indexingParams, facetRequests)
# and create a FacetsCollector to use in our facetted search:
facetsCollector = FacetsCollector.create(facetSearchParams, indexReader, taxoReader)
# perform documents search and facets accumulation
searcher.search(query, MultiCollector.wrap([topDocsCollector, facetsCollector]))
print "\nFound %d Documents for query=%s" % (topDocsCollector.totalHits,
query.toString().encode('utf-8'))
# Obtain facets results and print them
res = facetsCollector.getFacetResults()
i = 0
for facetResult in res:
print "Result #%d has %d descendants" % (i, facetResult.getNumValidDescendants())
print "Result #%d : %s" % (i, facetResult)
i += 1
return res
class Searcher:
def __init__(self, indexDir):
self.directory = SimpleFSDirectory(Paths.get(indexDir))
self.searcher = IndexSearcher(DirectoryReader.open(self.directory))
self.nameQueryParser = QueryParser('name', StandardAnalyzer())
self.nameQueryParser.setDefaultOperator(QueryParser.Operator.AND)
self.idQueryParser = QueryParser('id', StandardAnalyzer())
self.idQueryParser.setDefaultOperator(QueryParser.Operator.AND)
def find_by_name(self, name):
query = self.nameQueryParser.parse(name)
docs = self.searcher.search(query, 100).scoreDocs
tables = []
for scoreDoc in docs:
doc = self.searcher.doc(scoreDoc.doc)
table = dict((field.name(), field.stringValue())
for field in doc.getFields())
tables.append(table)
return tables
def find_by_id(self, id):
query = self.idQueryParser.parse(id)
docs = self.searcher.search(query, 100).scoreDocs
tables = []
for scoreDoc in docs:
doc = self.searcher.doc(scoreDoc.doc)
table = dict((field.name(), field.stringValue())
for field in doc.getFields())
tables.append(table)
return tables
class Lindex(object):
def __init__(self, index_dir_ngram='./ngram', index_dir_vocab='./vocab'):
self._index_dir_ngram = index_dir_ngram
self._index_dir_vocab = index_dir_vocab
fs = MMapDirectory.open(File(index_dir_ngram))
self._searcher_ngram = IndexSearcher(DirectoryReader.open(fs))
def __getitem__(self, ngram):
pass
def search_interactive(self):
doc = None
while True:
print()
print('Hit enter with no input to quit.')
command = raw_input('Query:')
if command == '':
return doc
print()
print('Searching for:'.format(command))
query = TermQuery(Term('ngram', command))
hits = self._searcher_ngram.search(query, 1000).scoreDocs
print('{} total matching documents.'.format(len(hits)))
for i, hit in enumerate(hits):
doc = self._searcher_ngram.doc(hit.doc)
print('doc {}:'.format(i))
for f in doc:
print(type(f))
print('\t{}'.format(f.toString().encode('utf-8')))
# print 'text:', doc.get("text")
if i >= 30:
break
def search_interactive_history(self):
doc = None
while True:
print()
print('Hit enter with no input to quit.')
command = raw_input('Query:')
if command == '':
return doc
print()
print('Searching for:'.format(command))
query = TermQuery(Term('history', command))
hits = self._searcher_ngram.search(query, 1000).scoreDocs
print('{} total matching documents.'.format(len(hits)))
for i, hit in enumerate(hits):
doc = self._searcher_ngram.doc(hit.doc)
print('doc {}:'.format(i))
for f in doc:
print(type(f))
print('\t{}'.format(f.toString().encode('utf-8')))
# print 'text:', doc.get("text")
if i >= 30:
break
def lucene_retrieval_multifield(q_string, q_class, feature_type, use_BM25=False):
"""
multifield: different query string for different field
not same word on different field
:param q_string:
:param feature_type:
:param use_BM25:
:return: retrieval_scores for each question-answer pair
"""
index = set_lucene_index['ind'] # nonlocal variable index
def retrieval_scores(hists):
"""
return sorted document+score by score
:param hists:
"""
def doc_score(hists):
"""
return doc_name & score
:param hists:
"""
for h in hists:
# docID = h.doc
# doc = searcher.doc(docID)
# file_name = doc.get("corpus_name")
# doc_name = doc.get("doc_name")
# text = doc.get("text")
score = h.score
# yield (file_name, doc_name, score, text)
yield score
doc_score_list = list(doc_score(hists))
return map(lambda f: f(doc_score_list), feature_type) # feature_type is a list of function
text_query = QueryParser(version, 'text', analyzer).parse(QueryParser.escape(q_string))
subject_query = QueryParser(version, 'corpus_name', analyzer).parse(QueryParser.escape(q_class))
query = BooleanQuery()
# BooleanClause.Occur
# MUST implies that the keyword must occur
# SHOULD implies that the keyword SHOULD occur
query.add(text_query, BooleanClause.Occur.SHOULD)
query.add(subject_query, BooleanClause.Occur.SHOULD)
# search
reader = IndexReader.open(index)
searcher = IndexSearcher(reader)
if use_BM25:
searcher.setSimilarity(BM25Similarity(k1=1.5, b=0.75)) # todo: BM25 parameters
collector = TopScoreDocCollector.create(hitsPerPage, True)
searcher.search(query, collector)
hs = collector.topDocs().scoreDocs # hists
results = retrieval_scores(hs)
# reader.close()
return results # retrieval_scores for each question-answer pair
class QAsearcher(object):
def __init__(self, dir):
self.dir = dir
self.lReader = DirectoryReader.open(
FSDirectory.open(Paths.get(self.dir)))
self.lSearcher = IndexSearcher(self.lReader)
# def get_collection_size(self):
# return self.lReader.numDocs()
def doc_search(self, keywords):
analyzer = StandardAnalyzer()
parser = QueryParser('Title', analyzer)
query = parser.parse(keywords)
try:
collector = TopScoreDocCollector.create(3000)
self.lSearcher.search(query, collector)
hits = collector.topDocs().scoreDocs
except RuntimeError:
print "Score docoment run fail"
self.hits = hits
return hits
def print_result(self):
temp = []
for i in self.hits:
# print "\nResult " + str(j) + "\tDocID: " + str(i.doc) + "\t Score: " + str(i.score)
try:
temp.append([
self.lReader.document(i.doc).get("Title"),
int(self.lReader.document(i.doc).get("AnswerCount"))
])
except RuntimeError:
print "Search fail"
sort = np.array(temp, dtype=object)
sort1000 = sort[np.argsort(-sort[:, 1])]
show = 'The top 10 question is \n'
for i in range(0, 10):
show = show + '\n' + 'The ' + str(i + 1) + ' question is: ' + str(
sort1000[i, 0]) + '\nThe number of answer is ' + str(
sort1000[i, 1]) + '\n'
text = '\n'.join(show)
easygui.textbox(title='6223 Top Question', text=show)
def close(self):
try:
if (self.lReader != None):
self.lReader.close()
except RuntimeError:
print "Close reader fail"
class SearchIndex(object):
def __init__(self):
vm_env = lucene.getVMEnv()
vm_env.attachCurrentThread()
indexDir = SimpleFSDirectory(File(app.config['INDEX_PATH']))
self.searcher = IndexSearcher(DirectoryReader.open(indexDir))
self.analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
self.parser = QueryParser(Version.LUCENE_CURRENT, "contents",
self.analyzer)
def search(self, q, page=1, duplicates=False):
query = self.parser.parse(q)
if not duplicates:
query = self.addDuplicatesQuery(query)
perPage = 10
start = (page - 1) * perPage
results = TopScoreDocCollector.create(1000, True)
self.searcher.search(query, results)
highlighter = Highlighter(QueryScorer(query))
highlighter.setTextFragmenter(SimpleFragmenter(40))
docs = []
for scoreDoc in results.topDocs(start, perPage).scoreDocs:
doc = self.searcher.doc(scoreDoc.doc)
tokenStream = self.analyzer.tokenStream(
"contents", StringReader(doc['contents']))
highlight = highlighter.getBestFragments(tokenStream,
doc['contents'], 3, "...")
docs.append({
'title': doc['title'],
'url': doc['url'],
'duplicate': doc['duplicate'],
'highlight': highlight
})
del self.searcher
totalPages = int(math.ceil(results.getTotalHits() / float(perPage)))
return totalPages, docs
def addDuplicatesQuery(self, query):
not_duplicate = TermQuery(Term('duplicate', 'false'))
booleanQuery = BooleanQuery()
booleanQuery.add(not_duplicate, BooleanClause.Occur.MUST)
booleanQuery.add(query, BooleanClause.Occur.MUST)
return booleanQuery
class TextSearcher:
def __init__(self, fs_directory):
directory = SimpleFSDirectory(Paths.get(fs_directory))
self.index_reader = DirectoryReader.open(directory)
self.searcher = IndexSearcher(DirectoryReader.open(directory))
self.analyzer = StandardAnalyzer()
self.query = None
self.lucene_dictionary = LuceneDictionary(self.index_reader,
'contents')
self.analyzer = StandardAnalyzer()
self.formatter = SimpleHTMLFormatter()
self.hits = None
def search(self, searchtext):
if searchtext is None:
return 0
self.query = QueryParser("contents", self.analyzer).parse(searchtext)
score_docs = self.searcher.search(self.query, 50).scoreDocs
print("%s total matching documents." % len(score_docs))
return len(score_docs)
def find_documents(self, search_text):
self.query = QueryParser("contents", self.analyzer).parse(search_text)
self.hits = self.searcher.search(self.query, 50)
return self.hits
def get_document(self, document_id):
return self.searcher.doc(document_id)
def get_current_query(self):
return self.query
def get_highlighted_hits(self):
extracted_fragments = []
scorer = QueryScorer(self.query)
fragmenter = SimpleSpanFragmenter(scorer, 10)
highlighter = Highlighter(self.formatter, scorer)
highlighter.setTextFragmenter(fragmenter)
for hit in self.hits.scoreDocs:
document = self.searcher.doc(hit.doc)
stream = TokenSources.getAnyTokenStream(self.index_reader, hit.doc,
'contents', self.analyzer)
best_fragments = highlighter.getBestFragments(
stream, document.get('contents'), 10)
for fragment in best_fragments:
print('fragment: ', fragment)
extracted_fragments.append((hit.doc, best_fragments))
return extracted_fragments
class SearchIndex(object):
def __init__(self):
vm_env = lucene.getVMEnv()
vm_env.attachCurrentThread()
indexDir = SimpleFSDirectory(File(app.config['INDEX_PATH']))
self.searcher = IndexSearcher(DirectoryReader.open(indexDir))
self.analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
self.parser = QueryParser(Version.LUCENE_CURRENT, "contents", self.analyzer)
def search(self, q, page = 1, duplicates = False):
query = self.parser.parse(q)
if not duplicates:
query = self.addDuplicatesQuery(query)
perPage = 10
start = (page - 1) * perPage
results = TopScoreDocCollector.create(1000, True)
self.searcher.search(query, results)
highlighter = Highlighter(QueryScorer(query))
highlighter.setTextFragmenter(SimpleFragmenter(40))
docs = []
for scoreDoc in results.topDocs(start, perPage).scoreDocs:
doc = self.searcher.doc(scoreDoc.doc)
tokenStream = self.analyzer.tokenStream("contents", StringReader(doc['contents']))
highlight = highlighter.getBestFragments(tokenStream, doc['contents'], 3, "...")
docs.append({
'title': doc['title'],
'url': doc['url'],
'duplicate': doc['duplicate'],
'highlight': highlight}
)
del self.searcher
totalPages = int(math.ceil(results.getTotalHits()/float(perPage)))
return totalPages, docs
def addDuplicatesQuery(self, query):
not_duplicate = TermQuery(Term('duplicate', 'false'))
booleanQuery = BooleanQuery()
booleanQuery.add(not_duplicate, BooleanClause.Occur.MUST)
booleanQuery.add(query, BooleanClause.Occur.MUST)
return booleanQuery
def lucene_retrieval(q_string, feature_type, use_BM25=False):
"""
:param q_string:
:param feature_type:
:param use_BM25:
:return: retrieval_scores for each question-answer pair
"""
index = set_lucene_index['ind'] # nonlocal variable index
def retrieval_scores(hists):
"""
return sorted document+score by score
:param hists:
"""
def doc_score(hists):
"""
return doc_name & score
:param hists:
"""
for h in hists:
# docID = h.doc
# doc = searcher.doc(docID)
# file_name = doc.get("corpus_name")
# doc_name = doc.get("doc_name")
# text = doc.get("text")
score = h.score
# yield (file_name, doc_name, score, text)
yield score
doc_score_list = list(doc_score(hists))
return map(lambda f: f(doc_score_list),
feature_type) if len(doc_score_list) != 0 else [0] * len(
feature_type) # feature_type is a list of function
# escape special characters via escape function
query = QueryParser(version, 'text',
analyzer).parse(QueryParser.escape(q_string))
# search
reader = IndexReader.open(index)
searcher = IndexSearcher(reader)
if use_BM25:
searcher.setSimilarity(BM25Similarity(k1=1.5,
b=0.75)) # todo: BM25 parameters
collector = TopScoreDocCollector.create(hitsPerPage, True)
searcher.search(query, collector)
hs = collector.topDocs().scoreDocs # hists
results = retrieval_scores(hs)
# reader.close()
return results # retrieval_scores for each question-answer pair
def lucene_retrieval(q_string, feature_type, use_BM25=False):
"""
:param q_string:
:param feature_type:
:param use_BM25:
:return: retrieval_scores for each question-answer pair
"""
index = set_lucene_index['ind'] # nonlocal variable index
def retrieval_scores(hists):
"""
return sorted document+score by score
:param hists:
"""
def doc_score(hists):
"""
return doc_name & score
:param hists:
"""
for h in hists:
# docID = h.doc
# doc = searcher.doc(docID)
# file_name = doc.get("corpus_name")
# doc_name = doc.get("doc_name")
# text = doc.get("text")
score = h.score
# yield (file_name, doc_name, score, text)
yield score
doc_score_list = list(doc_score(hists))
return map(lambda f: f(doc_score_list), feature_type) if len(doc_score_list) != 0 else [0]*len(feature_type) # feature_type is a list of function
# escape special characters via escape function
query = QueryParser(version, 'text', analyzer).parse(QueryParser.escape(q_string))
# search
reader = IndexReader.open(index)
searcher = IndexSearcher(reader)
if use_BM25:
searcher.setSimilarity(BM25Similarity(k1=1.5, b=0.75)) # todo: BM25 parameters
collector = TopScoreDocCollector.create(hitsPerPage, True)
searcher.search(query, collector)
hs = collector.topDocs().scoreDocs # hists
results = retrieval_scores(hs)
# reader.close()
return results # retrieval_scores for each question-answer pair
def search(self, input_query=None, max_answers=10): ''' Searches the given query in the index ''' if input_query is None: return None base_dir = '.' directory = SimpleFSDirectory(File(os.path.join(base_dir, self.index_dir))) searcher = IndexSearcher(DirectoryReader.open(directory)) analyzer = StandardAnalyzer(Version.LUCENE_CURRENT) # query = QueryParser(Version.LUCENE_CURRENT, "title", analyzer).parse(input_query) parser = MultiFieldQueryParser(Version.LUCENE_CURRENT, (self._posts_fields + self._answer_fields), analyzer) query = MultiFieldQueryParser.parse(parser, input_query) scoreDocs = searcher.search(query, max_answers).scoreDocs print "%s total matching documents." % len(scoreDocs) docs = [] for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) doc_dict = dict((field.name(), field.stringValue()) for field in doc.getFields()) docs.append(doc_dict) # print doc return docs
def retrieve(indexdir, queries):
lucene.initVM()
f = open("results_lucene.txt", "w")
analyzer = StandardAnalyzer(Version.LUCENE_4_10_1)
reader = IndexReader.open(SimpleFSDirectory(File(indexdir)))
searcher = IndexSearcher(reader)
fields = ["title", "abstract", "authors"]
st = PorterStemmer()
for id, q in queries.iteritems():
query = q
tokenizer = RegexpTokenizer(r'\w+')
qwords = tokenizer.tokenize(query)
qwords_k = [st.stem(q) for q in qwords]
query = " ".join(qwords_k)
parser = MultiFieldQueryParser(Version.LUCENE_CURRENT, fields, analyzer)
parser.setDefaultOperator(QueryParserBase.OR_OPERATOR)
query = MultiFieldQueryParser.parse(parser, query)
MAX = 1000
hits = searcher.search(query, MAX)
# print "Found %d document(s) that matched query '%s':" % (hits.totalHits, query)
for i, hit in enumerate(hits.scoreDocs):
f.write("%s Q0 %s %s %s G17R3\n" % (id, hit.doc+1, i+1, hit.score))
# print hit.doc+1, hit.score
# doc = searcher.doc(hit.doc)
# print doc.get("authors").encode("utf-8")
f.close()
def search(self, field, text):
"""
search text within indexed data
input:
field fieldname of the value that will be indexed
text text to search
output:
hits return a list of hits
"""
results = []
idx_reader = DirectoryReader.open(self.directory)
idx_searcher = IndexSearcher(idx_reader)
# parse query
parser = AnalyzingQueryParser(Version.LUCENE_CURRENT, field, self.analyser)
query = parser.parse(text)
# search
hits = idx_searcher.search(query, 1000).scoreDocs.tolist()
for hit in hits:
doc = idx_searcher.doc(hit.doc)
score = hit.score
title = doc.get(field)
url = doc.get("url")
results.append((score, url, title))
return results
def find_all_text_occurrences(objects: list) -> (dict, DirectoryReader):
docs_lookup = dict()
# noinspection PyUnresolvedReferences
lucene.initVM(initialheap='32m', maxheap='4G')
file = Paths.get("D:\GitHubD\BREDS\wiki_text_index\WIKI_TEXT")
dir = FSDirectory.open(file)
reader = DirectoryReader.open(dir)
searcher = IndexSearcher(reader)
parser = QueryParser('contents', StandardAnalyzer())
logging.warning(
'FOR MULTI-WORD OBJECTS, ALL DOCUMENTS WITH BOTH TERMS SEPARATELY WILL BE RETRIEVED'
)
for object in objects:
tokens = object.split(' ')
doc_sets = []
for token in tokens:
q = parser.parse(f'"{token}"')
# TODO maybe use minimum score
topdocs = searcher.search(q, 99999999)
results = set([topdoc.doc for topdoc in topdocs.scoreDocs])
doc_sets.append(results)
docs_lookup[object] = set.intersection(*doc_sets)
return docs_lookup, reader
def search(self, index_dir): # Get handle to index directory directory = SimpleFSDirectory(File(index_dir)) # Creates a searcher searching the provided index. ireader = DirectoryReader.open(directory) # Implements search over a single IndexReader. # Use a single instance and use it across queries # to improve performance. searcher = IndexSearcher(ireader) # Get the analyzer analyzer = StandardAnalyzer(Version.LUCENE_CURRENT) # Constructs a query parser. We specify what field to search into. queryParser = QueryParser(Version.LUCENE_CURRENT, FIELD_CONTENTS, analyzer) # Create the query query = queryParser.parse(self.query) # Run the query and get top 50 results topDocs = searcher.search(query, self.retrieve_count) # Get top hits scoreDocs = topDocs.scoreDocs doc_ids = [] for scoreDoc in scoreDocs: doc = searcher.doc(scoreDoc.doc) doc_ids.append(doc.get(FIELD_PATH)) return [int(item) for item in doc_ids]
def func2(command):
STORE_DIR = "index1"
vm_env = lucene.getVMEnv()
vm_env.attachCurrentThread()
directory = SimpleFSDirectory(File(STORE_DIR))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = WhitespaceAnalyzer(Version.LUCENE_CURRENT)
res = []
if command == '':
return
query = QueryParser(Version.LUCENE_CURRENT, "zhuliao",
analyzer).parse(command)
scoreDocs = searcher.search(query, 9).scoreDocs
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
try:
res.append([
doc.get("name"),
doc.get("collect_num"),
doc.get("zhuliao").split(' '),
doc.get("zuofa").split('\n'),
doc.get("img_url"),
doc.get("url")
])
except:
pass
res1 = []
for i in res:
i[1] = int(i[1])
res1.append(tuple(i))
res2 = sorted(res1, cmp=None, key=lambda x: x[1], reverse=True)
return res2
def query(self, data):
if self.fil.exists():
searcher = IndexSearcher(DirectoryReader.open(self.d))
query = QueryParser(
Version.LUCENE_30,
"id",
self.analyzer).parse(
data['query'])
hits = searcher.search(query, 100000)
results = {}
results['totalHits'] = hits.totalHits
results['hits'] = {}
for hit in hits.scoreDocs:
record = {}
doc = searcher.doc(hit.doc)
fields = doc.getFields()
record['score'] = hit.score
for field in fields:
if field.name() != "id":
record[field.name()] = field.stringValue()
results['hits'][doc.get('id')] = record
searcher.getIndexReader().close()
return results
def func1(command):
global vm_env
STORE_DIR = "index"
vm_env = lucene.getVMEnv()
vm_env.attachCurrentThread()
#lucene.initVM(vmargs=['-Djava.awt.headless=true'])
# base_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
directory = SimpleFSDirectory(File(STORE_DIR))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = WhitespaceAnalyzer(Version.LUCENE_CURRENT)
if command == '':
return []
command_list = jieba.cut(command)
command = " ".join(command_list)
query = QueryParser(Version.LUCENE_CURRENT, "contents",
analyzer).parse(command)
scoreDocs = searcher.search(query, 50).scoreDocs
result = []
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
doct = {
'title': doc.get("title"),
'url': doc.get("url"),
"sentence": doc.get("sentence")
}
result.append(doct)
del searcher
return result
class Searcher:
#comment out to run searcher by itself
lucene.initVM(vmargs=['-Djava.awt.headless=true'])
def __init__(self, indexDir):
f = Paths.get(indexDir)
self._dir = SimpleFSDirectory(f)
self._indexSearcher = IndexSearcher(DirectoryReader.open(self._dir))
self._weights = HashMap()
self._weights.put(FIELDS[0], 1)
self._weights.put(FIELDS[1], 0.2)
def search(self, query):
SHOULD = BooleanClause.Occur.SHOULD
q = MultiFieldQueryParser.parse(query, FIELDS, [SHOULD, SHOULD],
StandardAnalyzer())
# print(q.toString())
topHits = 100
scores = self._indexSearcher.search(q, topHits).scoreDocs
results = []
for i in range(10):
doc = self._indexSearcher.doc(scores[i].doc)
results.append(i + 1, scores[i].doc, doc.get("filename"),
doc.get("contents"))
# print(i+1)
# print("Score: ", scores[i].doc)
# print("Title: ", doc.get("filename"))
# print("Contents: ", doc.get("contents"))
return results
def retrieve_sents(self):
indexDir = self.indexDir
query = self.query
sent_ind_list = []
# template = CustomTemplate(format)
fsDir = SimpleFSDirectory(Paths.get(indexDir))
# print indexDir
searcher = IndexSearcher(DirectoryReader.open(fsDir))
analyzer = StandardAnalyzer()
parser = QueryParser("contents", analyzer)
parser.setDefaultOperator(QueryParser.Operator.OR)
query = parser.parse(query)
# print query
start = datetime.now()
scoreDocs = searcher.search(query, 50).scoreDocs
duration = datetime.now() - start
# print query
if self.stats:
print >> sys.stderr, "Found %d sentences (in %s) that matched query '%s':" % (
len(scoreDocs), duration, query)
for scoreDoc in scoreDocs:
# print scoreDoc.doc
# doc = searcher.doc(scoreDoc.doc)
sent_ind_list.append(scoreDoc.doc)
return sent_ind_list
def GET(self):
command = web.input().command.encode('utf-8')
initvm.vm_env.attachCurrentThread()
STORE_DIR = "jdindex"
directory = SimpleFSDirectory(File(STORE_DIR))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
query = QueryParser(Version.LUCENE_CURRENT, "contents",
analyzer).parse(command)
scoreDocs = searcher.search(query, 20).scoreDocs
finalDocs = []
for i, scoreDoc in enumerate(scoreDocs):
doc = searcher.doc(scoreDoc.doc)
title = doc.get("title").strip('\n')
if title not in finalDocs:
finalDocs.append(title)
web.header('content-type', 'text/json')
data = {}
data['q'] = command
data['p'] = 'false'
data['s'] = finalDocs
return 'fn(' + json.dumps(data) + ');'
def run_music(ID):
STORE_DIR = "index"
vm_env = lucene.getVMEnv()
vm_env.attachCurrentThread()
directory = SimpleFSDirectory(File(STORE_DIR))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
query = QueryParser(Version.LUCENE_CURRENT, "id", analyzer).parse(ID)
scoreDocs = searcher.search(query, 1).scoreDocs
try:
scoreDoc = scoreDocs[0]
except:
return None
doc = searcher.doc(scoreDoc.doc)
item = []
item.append(doc.get("song_title").encode('utf-8'))
item.append(doc.get('song_url'))
item.append(doc.get("singer").encode('utf-8'))
item.append(doc.get("album").encode('utf-8'))
item.append(doc.get("album_pic"))
item.append(doc.get("album_genre").encode('utf-8'))
item.append(doc.get("lyrics").encode('utf-8'))
sim_str = doc.get("similar").encode('utf-8')
sim_list = sim_str.split('+')
for i in range(3):
sim_list[i] = sim_list[i].split('*')
item.append(sim_list)
del searcher
return item
class LuceneSearch():
def __init__(self, lucene_index_dir='/data/zjy/csqa_data/lucene_dir/'):
lucene.initVM(vmargs=['-Djava.awt.headless=true'])
directory = SimpleFSDirectory(File(lucene_index_dir))
self.searcher = IndexSearcher(DirectoryReader.open(directory))
self.num_docs_to_return = 5
self.ireader = IndexReader.open(directory)
def search(self, value):
query = TermQuery(Term("wiki_name", value.lower()))
# query = BooleanQuery()
# query.add(new TermQuery(Term("wikidata_name",v)),BooleanClause.Occur.SHOULD)
# query.add(new TermQuery(Term("wikidata_name",v)),BooleanClause.Occur.SHOULD)
scoreDocs = self.searcher.search(query,
self.num_docs_to_return).scoreDocs
pids = []
for scoreDoc in scoreDocs:
doc = self.searcher.doc(scoreDoc.doc)
for f in doc.getFields():
print f.name(), ':', f.stringValue(), ', '
if (f.name() == "wiki_id"):
pids.append(f.stringValue())
print ''
print '-------------------------------------\n'
return pids
def search(self):
''' Searches the given query in the index '''
lucene.initVM(vmargs=['-Djava.awt.headless=true'])
# print 'lucene', lucene.VERSION
# base_dir = os.path.dirname(os.path.abspath('.'))
base_dir = '.'
directory = SimpleFSDirectory(File(os.path.join(base_dir, self.index_dir)))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
while True:
print
print "Hit enter with no input to quit."
command = raw_input("Query:")
if command == '':
return
print
print "Searching for:", command
query = QueryParser(Version.LUCENE_CURRENT, "title",
analyzer).parse(command)
scoreDocs = searcher.search(query, 50).scoreDocs
print "%s total matching documents." % len(scoreDocs)
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
# print 'path:', doc.get("path"), 'name:', doc.get("name")
print doc
def search_img(output):
STORE_DIR = "WebPageIndex"
directory = SimpleFSDirectory(File(STORE_DIR))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = WhitespaceAnalyzer(Version.LUCENE_CURRENT)
results = []
results.append([])
for num in output:
imgnum = str(num)
query = QueryParser(Version.LUCENE_CURRENT, "imgnum",
analyzer).parse(imgnum)
scoreDocs = searcher.search(query, 1).scoreDocs
for scoreDoc in scoreDocs:
result = []
doc = searcher.doc(scoreDoc.doc)
result.append(doc.get("album"))
result.append(doc.get("subalbum"))
result.append(doc.get("singer"))
result.append(doc.get("url"))
result.append(doc.get("reviews"))
result.append(doc.get("imgurl"))
result.append(doc.get("imgnum"))
results.append(result)
del searcher
return results
def func(command):
vm_env = lucene.getVMEnv()
vm_env.attachCurrentThread()
# ------------ #
STORE_DIR = "index"
directory = SimpleFSDirectory(File(STORE_DIR))
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
searcher = IndexSearcher(DirectoryReader.open(directory))
# ------------ #
p = get_d_dimensional_vector(command)
vp = get_vp(p)
query = QueryParser(Version.LUCENE_CURRENT, "Vector", analyzer).parse(vp)
scoreDocs = searcher.search(query, 200).scoreDocs
dict1 = {}
result = ""
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
rank = 0.6 * float(doc.get("Likes")) + 0.4 * float(doc.get("Views"))
ch = doc.get('Page_num') + ' '
ch += 'data/' + doc.get('Page_num') + '.jpg' + ' '
ch += doc.get('Page_link') + ' '
ch += doc.get('Views') + ' '
ch += doc.get('Likes') + ' '
tmp_alt = doc.get('Img_alt')
tmp_alt = '_'.join(tmp_alt.split())
ch += tmp_alt
dict1[ch] = rank
res_list = sorted(dict1.items(), key = lambda item:item[1], reverse = True)
for i in res_list:
result += i[0]
result += ' '
del searcher
del analyzer
return result
class LuceneSearcher(object):
fields = ['id', 'text', 'types']
def __init__(self, db_path):
directory = SimpleFSDirectory(File(db_path))
reader = DirectoryReader.open(directory)
self.searcher = IndexSearcher(reader)
self.analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
logger.info("Loaded DB from %s with %d documents: ",
db_path, reader.numDocs())
def search(self, query, max_matches=1000):
query = VALID_CHARS_PATTERN.sub(' ', query)
logger.debug("Searching for %s", query)
query = QueryParser(Version.LUCENE_CURRENT, "text",
self.analyzer).parse(query)
score_docs = self.searcher.search(query, max_matches).scoreDocs
logger.debug("%s total matching documents.",
len(score_docs))
docs = [self.searcher.doc(d.doc) for d in score_docs]
return [self.convert_to_dict(doc) for doc in docs]
def convert_to_dict(self, doc):
return {field: doc.get(field) for field in self.fields}
class SSQA_S_Searcher:
def __init__(self, indexDir, analyzer):
lucene.initVM()
self.reader = DirectoryReader.open(indexDir)
self.searcher = IndexSearcher(self.reader)
self.searcher.setSimilarity(mySimilarity())
self.analyzer = analyzer
logger.debug("Search similarity func: {}".format(
self.searcher.getSimilarity()))
def search(self, query_text, top_n):
query_text = query_text.strip()
query = QueryParser("content", self.analyzer).parse(
QueryParser.escape(query_text.strip()))
scoreDocs = self.searcher.search(query, top_n).scoreDocs
count = 0
out_list = []
for scoreDoc in tqdm(scoreDocs):
docIndex = scoreDoc.doc
doc = self.searcher.doc(docIndex)
log_debug(doc, logger)
log_debug(self.searcher.explain(query, docIndex), logger)
out_list.append(doc['content'])
count += 1
logger.info("Added {} sentences".format(count))
return out_list
def close(self):
self.reader.close()
def searcher(directory, analyzer, queries_file):
lines = queries_file.readlines()
length = len(lines)
a_query = ''
query_counter = 0
log = open("log.txt", "a")
for line_number in range(length):
if lines[line_number].startswith("<num>"):
query_id = lines[line_number][14:].strip()
elif lines[line_number].startswith("<desc>"):
a_query = lines[line_number + 1].strip()
a_query = stop_words(a_query)
if a_query != '':
# searching the index
reader = DirectoryReader.open(directory)
searcher = IndexSearcher(reader)
# parse the query
parser = QueryParser("DocParagraph", analyzer)
query = parser.parse(a_query)
# return 50 queries are required by the assignment
hits = searcher.search(query, 50).scoreDocs
# rank counter 1 through 50
rank_counter = 1
for hit in hits:
result = searcher.doc(hit.doc)
# write search result to log text file
to_log = str(query_id) + " " + "Q" + str(
query_counter) + " " + str(result.get(
"DocID")) + " " + str(rank_counter) + " " + str(
hit.score) + " " + "Alex's" + "\n"
log.write(to_log)
rank_counter += 1
query_counter += 1
a_query = ''
log.close()
def SearchImgCommand(command):
initvm.vm_env.attachCurrentThread()
STORE_DIR = "jdindex"
directory = SimpleFSDirectory(File(STORE_DIR))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
contentCommand = ' '.join(jieba.cut(command))
query = QueryParser(Version.LUCENE_CURRENT, "contents",
analyzer).parse(contentCommand)
scoreDocs = searcher.search(query, 50).scoreDocs
Already = []
finalDocs = []
for i, scoreDoc in enumerate(scoreDocs):
doc = searcher.doc(scoreDoc.doc)
itemurl = doc.get("itemurl")
if itemurl not in Already:
oneDoc = {}
oneDoc['imgurl'] = doc.get("imgurl")
oneDoc['title'] = doc.get("title").strip('\n')
oneDoc['itemurl'] = itemurl
oneDoc['score'] = scoreDoc.score
finalDocs.append(oneDoc)
Already.append(itemurl)
return finalDocs
def retrieve(command):
try:
lucene.initVM(vmargs=['-Djava.awt.headless=true'])
except ValueError:
print "JVM running."
print 'lucene', lucene.VERSION
base_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
directory = SimpleFSDirectory(Paths.get(os.path.join(base_dir, INDEX_DIR)))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = StandardAnalyzer()
# to convert to AND query
command = re.sub(r' ', r' +', command)
command = "+" + command
print "Searching for:", command
query = QueryParser("contents", analyzer).parse(command)
print query
scoreDocs = searcher.search(query, 500).scoreDocs
print "%s total matching documents." % len(scoreDocs)
retrieved_docs = []
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
retrieved_docs.append(os.path.join(doc.get("path"), doc.get("name")))
del searcher
return retrieved_docs
class Searcher(object):
def __init__(self, **kwargs):
""" Initialize a new instance of the Searcher
:param count: The number of counts to return from a query
:param output: The output directory of the underlying index
"""
self.count = kwargs.get("count", 100)
self.output = kwargs.get("root", "index")
self.store = SimpleFSDirectory(File(self.output))
self.analyzer = StandardAnalyzer(Version.LUCENE_30)
self.searcher = IndexSearcher(DirectoryReader.open(self.store))
def search(self, query):
""" Given a query, apply it against the existing index.
:param query: The query to apply to the index
:returns: A generator of the matching documents
"""
query = QueryParser(Version.LUCENE_30, "data", self.analyzer).parse(query)
results = self.searcher.search(query, self.count)
for result in results.scoreDocs or []:
# logger.debug("%s %s %s", hit.score, hit.doc, hit.toString())
document = self.searcher.doc(result.doc)
yield document.get("path"), result.score
def search(self, field, text):
"""
search text within indexed data
input:
field fieldname of the value that will be indexed
text text to search
output:
hits return a list of hits
"""
results = []
idx_reader = DirectoryReader.open(self.directory)
idx_searcher = IndexSearcher(idx_reader)
# parse query
parser = AnalyzingQueryParser(Version.LUCENE_CURRENT, field,
self.analyser)
query = parser.parse(text)
# search
hits = idx_searcher.search(query, 1000).scoreDocs.tolist()
for hit in hits:
doc = idx_searcher.doc(hit.doc)
score = hit.score
title = doc.get(field)
url = doc.get("url")
results.append((score, url, title))
return results
def buscar(indexDir, args,options = None):
#lucene.initVM(vmargs=['-Djava.awt.headless=true'])
fsDir = SimpleFSDirectory(File(indexDir))
#print fsDir
#Criando buscador baseado no diretorio dos indices passados pelo usuario
searcher = IndexSearcher(DirectoryReader.open(fsDir))
#Analizador para filtro dos tokens
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
#print analyzer
#Criando um QueryParser usando por padrao contents
#Variavel com as restricoes da busca
parser = QueryParser(Version.LUCENE_CURRENT, "keywords", analyzer)
#print parser
parser.setDefaultOperator(QueryParser.Operator.AND)
#print args
#Juntando parametros passados com o valor do mesmo
command = ' +'.join(args)
#print command
query = parser.parse(command)
print query
#Criando um JArray com resultado da consulta
return searcher.search(query, 200).scoreDocs
def shourcut_retriever(keyword):
'''查询器：在简介ä¸æŸ¥è¯¢'''
global flag
if flag:
lucene.initVM()
flag = False
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
reader = IndexReader.open(SimpleFSDirectory(File("index/")))
searcher = IndexSearcher(reader)
query = QueryParser(Version.LUCENE_4_10_1, "shortcut",
analyzer).parse(keyword)
MAX = 20
hits = searcher.search(query, MAX)
print("Found %d document(s) that matched query '%s':" %
(hits.totalHits, query))
results = []
for hit in hits.scoreDocs:
print(hit.score, hit.doc, hit.toString())
doc = searcher.doc(hit.doc)
result = [doc.get('shortcut'), doc.get('url'), doc.get('name')]
print(doc.get('url'))
results.append(result)
return results
def run(command):
global vm_env
STORE_DIR = "index"
vm_env = lucene.getVMEnv()
vm_env.attachCurrentThread()
directory = SimpleFSDirectory(File(STORE_DIR))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
query = QueryParser(Version.LUCENE_CURRENT, "contents",
analyzer).parse(command)
scoreDocs = searcher.search(query, 10).scoreDocs
#print "%s total matching documents." % len(scoreDocs)
res = []
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
tmp = []
tmp.append([doc.get('name1'), doc.get('name2')])
tmp.append(doc.get("homepage"))
tmp.append(doc.get("intro"))
tmp.append(doc.get('logo'))
a = doc.get('goods')
a = a.split('\n')
for i in a:
tmp.append(i)
res.append(tmp)
return command, res
class Search():
def __init__(self, file_name):
lucene.initVM(vmargs=['-Djava.awt.headless=true'])
base_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
directory = SimpleFSDirectory(
Paths.get(os.path.join(base_dir, file_name)))
self.reader = DirectoryReader.open(directory)
self.searcher = IndexSearcher(self.reader)
self.analyzer = StandardAnalyzer()
self.error_count = 0
#import pdb; pdb.set_trace()
def search_one(self, line, search_type, return_count=1):
line = line.replace("?",
"").replace("OR",
"").replace("AND",
"").replace("NOT", "")
line = QueryParser.escape(line)
search_types = ["all", "context", "last", "response"]
query = QueryParser(search_types[search_type],
self.analyzer).parse(line)
scoreDocs = self.searcher.search(query, return_count).scoreDocs
if len(scoreDocs) == 0:
self.error_count += 1
print("Error in search: ", line)
return None
res = []
for scoreDoc in scoreDocs:
doc = self.searcher.doc(scoreDoc.doc)
score = scoreDoc.score
res.append((doc.get("all"), doc.get("context"), doc.get("last"),
doc.get("response"), score))
return res
def get_query_results(reader,query,n,field):
searcher = IndexSearcher(reader)
hits = searcher.search(query, n).scoreDocs
print("Found %d hits:" % len(hits))
for i, hit in enumerate(hits):
doc = searcher.doc(hit.doc)
print("%d. %s" % (i + 1, doc.get(field)))
def get_image_pmcid(pmcid, classes = ""):
fields = ["pmcid", "class"]
docs = []
location = web.__path__[0] + "/static/web/files/index/index.figures"
#lucene.initVM()
vm_env = lucene.getVMEnv()
vm_env.attachCurrentThread()
analyzer = StandardAnalyzer(Version.LUCENE_4_10_1)
reader = IndexReader.open(SimpleFSDirectory(File(location)))
searcher = IndexSearcher(reader)
# multi field query: http://stackoverflow.com/questions/2005084/how-to-specify-two-fields-in-lucene-queryparser
#query = MultiFieldQueryParser(Version.LUCENE_4_10_1, fields, analyzer)
# query.setDefaultOperator(QueryParserBase.AND_OPERATOR)
#query = query.parse(query, ('4175339','1'))
# query.parse(queryString)#"Shigella sonnei"
# query = QueryParser(Version.LUCENE_4_10_1, "abstract", analyzer).parse(queryString)#"Shigella sonnei"
MAX = 10000
#hits = searcher.search(query, MAX)
if classes == "all":
queryStr = "pmcid:(" + ' '.join(pmcid) +")"
else:
queryStr = "pmcid:(" + ' '.join(pmcid) +")" + " AND class:" + classes
query = QueryParser(Version.LUCENE_4_10_1, "pmcid",analyzer)#needed to build a custom query
q = query.parse(queryStr)
hits = searcher.search(q, MAX)
for hit in hits.scoreDocs:#should only be one
#print hit.score, hit.doc, hit.toString()
docs.append(searcher.doc(hit.doc))
return docs #This will return the image documents that belong to a pmcid(article)
def get_candidates(qatp):
if prm.create_index:
create_index()
lucene.initVM()
analyzer = StandardAnalyzer(Version.LUCENE_4_10_1)
reader = IndexReader.open(SimpleFSDirectory(File(prm.index_folder)))
searcher = IndexSearcher(reader)
candidates = []
n = 0
for q,a,t,p in qatp:
if n % 100 == 0:
print 'finding candidates sample', n
n+=1
q = q.replace('AND','\\AND').replace('OR','\\OR').replace('NOT','\\NOT')
query = QueryParser(Version.LUCENE_4_10_1, "text", analyzer).parse(QueryParser.escape(q))
hits = searcher.search(query, prm.max_candidates)
c = []
for hit in hits.scoreDocs:
doc = searcher.doc(hit.doc)
c.append(doc.get("id"))
candidates.append(c)
return candidates
def main():
resultados = []
indice_vacio = False
if len(os.listdir("./lucene/index")) == 0:
indice_vacio = True
else:
consulta = request.args.get("consulta", None)
if consulta is not None:
directory = SimpleFSDirectory(Paths.get("./lucene/index"))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = SpanishAnalyzer()
query = QueryParser("texto", analyzer).parse(consulta)
scoreDocs = searcher.search(query, 10).scoreDocs
for sd in scoreDocs:
doc = searcher.doc(sd.doc)
resultados.append({
"url": direccion_base + doc.get("pdf"),
"titulo": doc.get("titulo")
})
return render_template("main.html",
lucene=lucene.VERSION,
indice_vacio=indice_vacio,
resultados=resultados)
def retriever(file_dir):
analyzer = WhitespaceAnalyzer()
reader = DirectoryReader.open(
SimpleFSDirectory(Paths.get(file_dir + "/lucene_index/")))
searcher = IndexSearcher(reader)
queryParser = QueryParser("code", analyzer)
BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE)
with open(file_dir + "/train/train.spl.src",
'r') as fso, open(file_dir + "/train/train.txt.tgt", 'r') as fsu:
sources = [line.strip() for line in fso.readlines()]
summaries = [line.strip() for line in fsu.readlines()]
with open(file_dir+"/test/test.ast.src") as ft, open(file_dir+"/test/test.ref.src.0", 'w') as fwo, \
open(file_dir+"/output/ast.out", 'w') as fws:
queries = [
re.sub("[\W\s]+|AND|NOT|OR", ' ', line.strip())
for line in ft.readlines()
]
for i, line in enumerate(queries):
print("query %d" % i)
query = queryParser.parse(QueryParser.escape(line))
hits = searcher.search(query, 1).scoreDocs
flag = False
for hit in hits:
doc = searcher.doc(hit.doc)
_id = eval(doc.get("id"))
flag = True
fwo.write(sources[_id] + '\n')
fws.write(summaries[_id] + '\n')
if not flag:
print(query)
print(hits)
exit(-1)
def search_docs(self, value, field="general_info"):
MAX_RESULTS = 1000
searcher = IndexSearcher(DirectoryReader.open(self.store))
query = QueryParser(Version.LUCENE_CURRENT, field,
self.analyzer).parse(value)
topDocs = searcher.search(query, MAX_RESULTS)
return [searcher.doc(hit.doc) for hit in topDocs.scoreDocs]
def lucene_retrieval(q_string, use_BM25=False):
"""
:param q_string:
:param use_BM25:
:return: retrieval_scores for each question-answer pair
"""
index = set_lucene_index['ind'] # nonlocal variable index
def doc_text(hists):
"""
return doc_name & score
:param hists:
"""
text = '_NONE_'
for h in hists:
docID = h.doc
doc = searcher.doc(docID)
# file_name = doc.get("corpus_name")
# doc_name = doc.get("doc_name")
text = doc.get("text")
#score = h.score
# yield (file_name, doc_name, score, text)
return text
result = '_NONE_'
# escape special characters via escape function
if q_string and q_string.strip(): # when pre-process answers, `none of the above` -> '' cause error here
#print(q_string)
query = QueryParser(version, 'text', analyzer).parse(QueryParser.escape(q_string))
# search
reader = IndexReader.open(index)
searcher = IndexSearcher(reader)
if use_BM25:
searcher.setSimilarity(BM25Similarity(k1=1.5, b=0.75)) # todo: BM25 parameters
collector = TopScoreDocCollector.create(hitsPerPage, True)
searcher.search(query, collector)
hs = collector.topDocs().scoreDocs # hists
result = doc_text(hs)
# reader.close()
return result # text: also nodes
def SearchQuery(queryString, fields, classification):
#if __name__ == "__main__":
#if __name__ == "retriever":
location = web.__path__[0] + "/static/web/files/index/index.articles"
#lucene.initVM()
vm_env = lucene.getVMEnv()
vm_env.attachCurrentThread()
analyzer = StandardAnalyzer(Version.LUCENE_4_10_1)
reader = IndexReader.open(SimpleFSDirectory(File(location)))
searcher = IndexSearcher(reader)
#multi field query: http://stackoverflow.com/questions/2005084/how-to-specify-two-fields-in-lucene-queryparser
query = MultiFieldQueryParser(Version.LUCENE_4_10_1, fields, analyzer)
#query.setDefaultOperator(QueryParserBase.AND_OPERATOR)
query = MultiFieldQueryParser.parse(query, queryString)
#query.parse(queryString)#"Shigella sonnei"
#query = QueryParser(Version.LUCENE_4_10_1, "abstract", analyzer).parse(queryString)#"Shigella sonnei"
MAX = 10000
hits = searcher.search(query, MAX)
print "Found %d document(s) that matched query '%s':" % (hits.totalHits, query)
paths = []
pmcids = []
documentDict = {}
for hit in hits.scoreDocs:
doc = searcher.doc(hit.doc)
pmcids.append(doc.get("pmcid"))
docDict = {"title" : doc.get("title")}#we can add any other field we want...
documentDict[doc.get("pmcid")] = docDict
#Where we get the images for all the pmcids
images = get_image_pmcid(pmcids, classification)#should take in pmcids and class
#create dictionary of images with pmcid being their key
imagesDict = {}
for img in images:
img_pmcid = img.get("pmcid")
if img_pmcid in imagesDict.keys():
imagesDict[img_pmcid].append(img.get("filepath") + "/" + img.get("figureid"))
else:
imagesDict[img_pmcid] = [(img.get("filepath") + "/" + img.get("figureid"))]
#for each pmcid, we will assign an image to it for the search results
for pmcid in pmcids:
if imagesDict:
docDict = documentDict[pmcid]
docDict["imgURL"] = imagesDict[pmcid][0]
documentDict[pmcid] = docDict
else:
docDict = documentDict[pmcid]
docDict["imgURL"] = "images/NoImageAvailable.jpg"
documentDict[pmcid] = docDict
#END - Where we get the images for all the pmcids
return documentDict
def perform_search(self, searchterm, results_per_page, page):
# if there is a field in the searchterm
"""if ":" in searchterm:
# processing a query
parser = QueryParser(Version.LUCENE_CURRENT, "content", self.analyzer)
parser.setDefaultOperator(QueryParser.Operator.AND)
query = parser.parse(searchterm)
else:
query = BooleanQuery()
query_title = TermQuery(Term("title", searchterm))
query_description = TermQuery(Term("description", searchterm))
query_content = TermQuery(Term("content", searchterm))
# BooleanClause.Occur.MUST for AND queries
query.add(query_title, BooleanClause.Occur.SHOULD)
query.add(query_description, BooleanClause.Occur.SHOULD)
query.add(query_content, BooleanClause.Occur.SHOULD)"""
# create QueryParser for each field to be searched
parser_title = QueryParser(Version.LUCENE_CURRENT, "title", self.analyzer)
parser_description = QueryParser(Version.LUCENE_CURRENT, "description", self.analyzer)
parser_content = QueryParser(Version.LUCENE_CURRENT, "content", self.analyzer)
# put fields together
query = BooleanQuery()
query.add(parser_title.parse(searchterm), BooleanClause.Occur.SHOULD)
query.add(parser_description.parse(searchterm), BooleanClause.Occur.SHOULD)
query.add(parser_content.parse(searchterm), BooleanClause.Occur.SHOULD)
# conducting search
searcher = IndexSearcher(DirectoryReader.open(self.store))
start = datetime.now()
hits = searcher.search(query, results_per_page + (results_per_page * page))
score_docs = hits.scoreDocs
count_results = hits.totalHits
duration = datetime.now() - start
# results to return
results = []
count = 0
for scoreDoc in score_docs:
# skip offset
if count < results_per_page * page:
count += 1
continue
count += 1
doc = searcher.doc(scoreDoc.doc)
table = dict((field.name(), field.stringValue()) for field in doc.getFields())
results.append(table)
return results, duration, count_results
def is_article_indexed(art_id, index='index'):
store = SimpleFSDirectory(File(index))
searcher = IndexSearcher(DirectoryReader.open(store))
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
query = QueryParser(Version.LUCENE_CURRENT, 'art_id', analyzer).parse(str(art_id))
docs = searcher.search(query, 1).scoreDocs
return len(docs) > 0
class Searcher:
def __init__(self, analyzer, indexDir):
self.mAnalyzer = analyzer
dir = SimpleFSDirectory(File(indexDir))
self.mIndexSearcher = IndexSearcher(DirectoryReader.open(dir))
def search(self, strQuery, fieldName, fieldName_2):
query = QueryParser(Version.LUCENE_CURRENT, fieldName, self.mAnalyzer).parse(strQuery)
docs = self.mIndexSearcher.search(query, 50).scoreDocs
if len(docs) < 1:
query = QueryParser(Version.LUCENE_CURRENT, fieldName_2, StandardAnalyzer(Version.LUCENE_CURRENT)).parse(
strQuery
)
docs = self.mIndexSearcher.search(query, 50).scoreDocs
return docs, query
class WikiPageIndex():
def __init__(self, index_dir):
#lucene.initVM(vmargs=['-Djava.awt.headless=true', '-Xmx4g'])
self.index_dir = index_dir
self.directory = SimpleFSDirectory(File(self.index_dir))
self.analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
self.config = IndexWriterConfig(Version.LUCENE_CURRENT, self.analyzer)
self.config.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
self.searcher = IndexSearcher(DirectoryReader.open(self.directory))
def createIndex(self):
self.writer = IndexWriter(self.directory, self.config)
if not os.path.exists(self.index_dir):
os.mkdir(self.index_dir)
def addDocumentToIndex(self, title, text):
doc = Document()
doc.add(Field("Title", title, Field.Store.YES, Field.Index.ANALYZED))
doc.add(Field("Text", text, Field.Store.YES, Field.Index.ANALYZED))
self.writer.addDocument(doc)
def closeIndex(self):
self.writer.commit()
self.writer.close()
def searchIndex(self, queryString, field="Text", max_results=100):
query = QueryParser(Version.LUCENE_CURRENT, field, self.analyzer).parse(queryString)
scoreDocs = self.searcher.search(query, max_results).scoreDocs
log.debug("Found {0} documents for query [{1}]".format(len(scoreDocs), queryString))
docs = []
for scoreDoc in scoreDocs:
doc = self.searcher.doc(scoreDoc.doc)
log.debug(WikiPageIndex.cleanWikiText(doc.get("Text")))
#print("title: {0}\ncontents: {1}".format(doc.get("Title"), doc.get("Text")[:70]))
docs.append(doc)
return docs
@staticmethod
def cleanWikiText(text):
text = text.encode('ascii', 'ignore')
text = re.sub('(\[\[.*?\]\]|\{\{.*?\}\}|\{\|.*?\|\})', '', text)
text = re.sub('[^\na-zA-Z0-9\n_-]+', ' ', text)
text = re.sub('([ \t]*[\n]+[ \t]*)+', '\n', text)
return text.strip()
def search(term, n_docs=10, index='index'):
store = SimpleFSDirectory(File(index))
searcher = IndexSearcher(DirectoryReader.open(store))
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
query = QueryParser(Version.LUCENE_CURRENT, 'art_body', analyzer).parse(term)
# str(query.getClass().toString()) == "class org.apache.lucene.search.TermQuery"
score_docs = searcher.search(query, n_docs).scoreDocs
return [(score_doc.score, unicode(searcher.doc(score_doc.doc).get('art_body'))) for score_doc in score_docs]
def retrieve_wiki(text_query, index_directory_name):
lucene.initVM()
directory = FSDirectory.open(File(index_directory_name))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
txt =text_query
query = QueryParser(Version.LUCENE_CURRENT, "contents", analyzer).parse(txt)
scoreDocs = searcher.search(query, 1000).scoreDocs
for scoreDoc in scoreDocs:
doc = searcher.doc(scoreDoc.doc)
yield doc.get('contents')
def main(indexDir, inputDir):
"""Creates a SQLite database with news linked to other news by at least one term, backed by a Lucene Index"""
lucene.initVM()
# Open index
logger.info("Opening Lucene index [%s]..." % indexDir)
dir = SimpleFSDirectory(File(indexDir))
analyzer = KeywordAnalyzer(Version.LUCENE_CURRENT)
reader = DirectoryReader.open(dir)
searcher = IndexSearcher(reader)
# Search documents
onlyfiles = [ f for f in listdir(inputDir) if isfile(join(inputDir, f)) and f.endswith('.json') ]
rels = list()
for f in onlyfiles:
journal_code = f.split('.')[0]
f = join(inputDir, f)
json_data = open(f)
data = json.load(json_data)
# The results collected after comparison
for entry in data:
url = entry['url']
date = entry['date']
title = entry['title']
logger.debug("Processing URL [%s] date [%s] - [%s]" % (url, date, title))
tt = nltk.word_tokenize(title)
tokens = []
for t in tt:
tokens.append(t.lower())
for token in tokens:
q = 'title: "%s" AND date: "%s" AND NOT journal: "%s" AND NOT url: "%s"' % (token, date, journal_code, url)
query = QueryParser(Version.LUCENE_CURRENT, "title", analyzer).parse(q)
hits = searcher.search(query, MAX_HITS)
logger.debug("Found %d document(s) that matched query '%s':" % (hits.totalHits, q))
for hit in hits.scoreDocs:
doc = searcher.doc(hit.doc)
logger.debug(doc)
rels.append({'left': url, 'token': token, 'right': doc.get('url')})
json_data.close()
with open('relationships.csv', 'wb') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
for rel in rels:
csvwriter.writerow([rel['left'].encode('utf8'), rel['token'].encode('utf8'), rel['right'].encode('utf8')])
class PyLucene(object):
def __init__(self):
if luceneImport:
self.lucene = True
else:
self.lucene = False
#Lucene connection
lucene.initVM()
indexDir = "texts/index"
directory = MMapDirectory(File(indexDir))
directory = DirectoryReader.open(directory)
self.analyzer = StandardAnalyzer(Version.LUCENE_30)
self.searcher = IndexSearcher(directory)
def query(self, terms = []):
query = QueryParser(Version.LUCENE_30, "text", self.analyzer).parse(" OR ".join(terms))
MAX = 1000
hits = self.searcher.search(query, MAX)
results = []
for hit in hits.scoreDocs:
doc = self.searcher.doc(hit.doc)
results.append([doc.get("doc_id").encode("utf-8"), doc.get("head").encode("utf-8")])
return results
def occurencies(self, term, morphs):
query = []
already = []
for morph in morphs:
query.append(morph)
#Sometime, when there is doubt about a term, because of xml hashing in Lucene, you would find twice a lemma like wordword
query.append(morph+morph)
results = self.query(query)
resultsReturned = []
for result in results:
if result[0] not in already:
resultsReturned.append(result)
already.append(result[0])
return resultsReturned, len(resultsReturned)
def chunk(self, occurency):
#Could be updated using the section information but could be only milesone
return occurency#, len(occurency)
def author_search(qry, limit):
helper.initPyLucene()
RNLP_ctxt = _get_rnlp_ctxt()
entry_map = RNLP_ctxt.get_entry_map()
rootdir = OUT_RAW_DIR
from org.apache.lucene.index import DirectoryReader
from org.apache.lucene.search import IndexSearcher
from org.apache.lucene.queryparser.classic import QueryParser
from org.apache.lucene.analysis.standard import StandardAnalyzer
from org.apache.lucene.store import FSDirectory
from org.apache.lucene.util import Version
from java.io import File
reader = DirectoryReader.open(FSDirectory.open(File(IDX_DIR)))
searcher = IndexSearcher(reader)
analyzer = StandardAnalyzer(Version.LUCENE_40)
field = 'contents'
parser = QueryParser(Version.LUCENE_40, field, analyzer);
query = parser.parse(qry);
print 'Searching for:', query.toString(field)
raw_results = searcher.search(query, limit)
hits = raw_results.scoreDocs
numTotalHits = raw_results.totalHits
print numTotalHits, 'total matching documents'
results = {}
for hit in hits:
doc = searcher.doc(hit.doc)
entry_id = doc.get('entry_id')
entry = entry_map.get(entry_id)
short_title = entry['short_title']
print(entry['prim_author'])
if qry in entry['prim_author'].lower():
fname = short_title + CONTENT_EXT
results[entry_id] = {'title': short_title, 'file': fname }
f = open ('/Users/Nelle/Documents/coding/text_analysis/newsvn/RenaissanceNLP/data/dataResults/authorPaths/' + qry + '.json', 'w')
f.write(json.dumps(results))
f.close()
return json.dumps(results)
def perform_search(self, searchterm):
# processing a query
parser = QueryParser(Version.LUCENE_CURRENT, "content", self.analyzer)
parser.setDefaultOperator(QueryParser.Operator.AND)
query = parser.parse(searchterm)
# conducting search
searcher = IndexSearcher(DirectoryReader.open(self.store))
start = datetime.now()
scoreDocs = searcher.search(query, 50).scoreDocs
duration = datetime.now() - start
print scoreDocs
print duration
def make_request(query, analyzer, index, qparser_regexp=None, max_results=100):
"""
:param query:
:param analyzer:
:param index:
:param qparser_regexp:
:param max_results:
:return:
"""
reader = IndexReader.open(SimpleFSDirectory(File(index)))
searcher = IndexSearcher(reader)
query = QueryParser(Version.LUCENE_30, "content", analyzer).parse(
query if not qparser_regexp else re.sub(qparser_regexp, " ", query)
)
hits = searcher.search(query, max_results)
return hits.scoreDocs
def make_request(query, analyzer, index, qparser_regexp=None, max_results=100):
""" Returns list of Score objects, which exposes `doc` (document ID in lucene) and `score` attrs
:param query:
:param analyzer:
:param index:
:param qparser_regexp:
:param max_results:
:return:
"""
reader = IndexReader.open(SimpleFSDirectory(File(index)))
searcher = IndexSearcher(reader)
query = QueryParser(Version.LUCENE_30, "content", analyzer).parse(
query if not qparser_regexp else re.sub(qparser_regexp, " ", query)
)
hits = searcher.search(query, max_results)
return hits.scoreDocs
def get_sorted_results(self, query):
SHOULD = BooleanClause.Occur.SHOULD
parsed_query = MultiFieldQueryParser.parse(Version.LUCENE_CURRENT, query, ['docno', 'content'], [SHOULD, SHOULD], self.analyzer)
reader = IndexReader.open(self.directory)
searcher = IndexSearcher(reader)
searcher.setSimilarity(BM25Similarity())
topDocs = searcher.search(parsed_query, 10)
j = 0
for i in topDocs.scoreDocs:
d = searcher.doc(i.doc)
print 'No. %02d: ' % (j + 1) + d['docno'] + ' ' + str(i.score)
j += 1
