def __init__(self, storeDir, analyzer):
if not os.path.exists(storeDir):
os.mkdir(storeDir)
store = SimpleFSDirectory(File(storeDir))
analyzer = LimitTokenCountAnalyzer(analyzer, 1048576)
config = IndexWriterConfig(Version.LUCENE_CURRENT, analyzer)
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
writer = IndexWriter(store, config)
self.indexDocs(writer)
ticker = Ticker()
print( 'optimizing index' ),
threading.Thread(target=ticker.run).start()
writer.commit()
writer.close()
ticker.tick = False
print( 'done' )
def __init__(self, root, storeDir, analyzer):
if not os.path.exists(storeDir):
os.mkdir(storeDir)
store = SimpleFSDirectory(File(storeDir))
analyzer = LimitTokenCountAnalyzer(analyzer, 1048576)
config = IndexWriterConfig(Version.LUCENE_CURRENT, analyzer)
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
writer = IndexWriter(store, config)
self.indexDocs(root, writer)
ticker = Ticker()
print('optimizing index'),
threading.Thread(target=ticker.run).start()
writer.commit()
writer.close()
ticker.tick = False
print('done')
def createIndex(cls, dataDir, indexDir, useCompound):
indexDir = SimpleFSDirectory(File(indexDir))
config = IndexWriterConfig(Version.LUCENE_CURRENT,
StandardAnalyzer(Version.LUCENE_CURRENT))
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
writer = IndexWriter(indexDir, config)
config = writer.getConfig()
mp = config.getMergePolicy()
if (LogMergePolicy.instance_(mp)):
mp.setUseCompoundFile(useCompound)
for dir, dirnames, filenames in os.walk(dataDir):
for filename in filenames:
if filename.endswith('.properties'):
cls.indexFile(writer, os.path.join(dir, filename), dataDir)
writer.commit()
writer.close()
def index(cls, indexDir, taxoDir):
"""Create an index, and adds to it sample documents and facets.
indexDir Directory in which the index should be created.
taxoDir Directory in which the taxonomy index should be created.
"""
# create and open an index writer
ver = lucene.Version.LUCENE_35
config = IndexWriterConfig(ver, WhitespaceAnalyzer(ver))
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
iw = IndexWriter(indexDir, config)
# create and open a taxonomy writer
taxo = DirectoryTaxonomyWriter(taxoDir,
IndexWriterConfig.OpenMode.CREATE)
# loop over sample documents
nDocsAdded = 0
nFacetsAdded = 0
for docNum in range(len(docTexts)):
# obtain the sample facets for current document
facets = categories[docNum]
facetList = [createCategoryPath(f) for f in facets]
# NOTE: setCategoryPaths() requires an Iterable, so need to convert the
# Python list in order to to pass a proper argument to setCategoryPaths.
# We use java.util.Arrays (via JCC) to create a Java List.
# see http://docs.oracle.com/javase/1.5.0/docs/api/java/util/Arrays.html#asList(T...)
facetList = lucene.Arrays.asList(facetList)
# NOTE: we could use lucene.collections here as well in order to convert our
# Python list to a Java based list using the JavaList class (JavaList implements
# java.util.List around a Python list instance it wraps):
# from lucene.collections import JavaList
# facetList = JavaList(facetList)
# we do not alter indexing parameters
# a category document builder will add the categories to a document once build() is called
categoryDocBuilder = CategoryDocumentBuilder(
taxo).setCategoryPaths(facetList)
# create a plain Lucene document and add some regular Lucene fields to it
doc = Document()
doc.add(
Field(TITLE, docTitles[docNum], Field.Store.YES,
Field.Index.ANALYZED))
doc.add(
Field(TEXT, docTexts[docNum], Field.Store.NO,
Field.Index.ANALYZED))
# invoke the category document builder for adding categories to the document and,
# as required, to the taxonomy index
categoryDocBuilder.build(doc)
# finally add the document to the index
iw.addDocument(doc)
nDocsAdded += 1
nFacetsAdded += facetList.size()
# end for
# commit changes.
# we commit changes to the taxonomy index prior to committing them to the search index.
# this is important, so that all facets referred to by documents in the search index
# will indeed exist in the taxonomy index.
taxo.commit()
iw.commit()
# close the taxonomy index and the index - all modifications are
# now safely in the provided directories: indexDir and taxoDir.
taxo.close()
iw.close()
print "Indexed %d documents with overall %d facets." % (nDocsAdded,
nFacetsAdded)
def getWriter(self, directory=None, analyzer=None, open_mode=None):
config = IndexWriterConfig(self.TEST_VERSION,
analyzer or LimitTokenCountAnalyzer(WhitespaceAnalyzer(Version.LUCENE_CURRENT), 10000)
)
config.setOpenMode(open_mode or IndexWriterConfig.OpenMode.CREATE)
return IndexWriter(directory or self.directory, config)
