if sourceData.tasksFile is None:
raise RuntimeError('No tasks file defined: -tasks [file]')
comp = competition.Competition(randomSeed=0)
index = comp.newIndex(
constants.TRUNK_CHECKOUT,
sourceData,
postingsFormat='Lucene90',
idFieldPostingsFormat='Memory',
grouping=False,
doDeletions=False,
addDVFields=True,
)
c = competition.Competitor('base', constants.TRUNK_CHECKOUT)
r = benchUtil.RunAlgs(constants.JAVA_COMMAND, False, False)
r.compile(c)
r.makeIndex(c.name, index, False)
cp = '%s' % r.classPathToString(benchUtil.getClassPath(c.checkout))
fip = '%s/index' % benchUtil.nameToIndexPath(index.getName())
modes = benchUtil.getArg('-mode', 'update', True)
docsPerSec = benchUtil.getArg('-dps', '1', True)
reopenPerSec = benchUtil.getArg('-rps', '0.2', True)
runTimeSec = benchUtil.getArg('-rts', 60, True)
numSearchThreads = benchUtil.getArg('-nst', 1,
True) # default to 1 search thread
numIndexThreads = benchUtil.getArg('-nit', constants.INDEX_NUM_THREADS,
True)
# MAX_BUFFERED_DOCS = 49774
# INDEXING_BUFFER_MB = -1
MAX_BUFFERED_DOCS = 5000
INDEXING_BUFFER_MB = -1
BODY_FIELD_TERM_VECTORS = True
# DOC_COUNT = 27625038
# DOC_COUNT = 100000
DOC_COUNT = 10000000
print('Compile luceneutil and %s/%s...' %
(constants.BASE_DIR, LUCENE_TRUNK_ROOT))
r = benchUtil.RunAlgs(JAVA_CMD, False)
c = competition.Competitor('foo', LUCENE_TRUNK_ROOT)
c.compile(r.classPathToString(r.getClassPath(c.checkout)))
while True:
print
print('%s: create index' % datetime.datetime.now())
shutil.rmtree(INDEX_PATH)
cmd = '%s -classpath "ROOT/lucene/build/core/classes/java:ROOT/lucene/build/core/classes/test:ROOT/lucene/build/sandbox/classes/java:ROOT/lucene/build/misc/classes/java:ROOT/lucene/build/facet/classes/java:/home/mike/src/lucene-c-boost/dist/luceneCBoost-SNAPSHOT.jar:ROOT/lucene/build/analysis/common/classes/java:ROOT/lucene/build/analysis/icu/classes/java:ROOT/lucene/build/queryparser/classes/java:ROOT/lucene/build/grouping/classes/java:ROOT/lucene/build/suggest/classes/java:ROOT/lucene/build/highlighter/classes/java:ROOT/lucene/build/codecs/classes/java:ROOT/lucene/build/queries/classes/java:lib/HdrHistogram.jar:build" perf.Indexer -dirImpl MMapDirectory -indexPath "%s" -analyzer StandardAnalyzerNoStopWords -lineDocsFile %s -docCountLimit %s -threadCount %d -maxConcurrentMerges 3 -dvfields -ramBufferMB %s -maxBufferedDocs %d -postingsFormat Lucene50 -waitForMerges -mergePolicy LogDocMergePolicy -facets Date -facetDVFormat Lucene50 -idFieldPostingsFormat Memory'.replace(
'ROOT', '%s/%s' % (constants.BASE_DIR, LUCENE_TRUNK_ROOT)) % (
JAVA_CMD, INDEX_PATH, LINE_DOCS_FILE, DOC_COUNT, INDEX_THREADS,
INDEXING_BUFFER_MB, MAX_BUFFERED_DOCS)
if USE_CMS:
cmd += ' -useCMS'