def setUp(self):
super(DeDupFilterCollectorTest, self).setUp()
self._reactor = CallTrace('reactor')
settings = LuceneSettings(commitCount=1, verbose=False)
self.lucene = Lucene(self.tempdir,
reactor=self._reactor,
settings=settings)
def luceneAndReaderConfig(defaultLuceneSettings, httpRequestAdapter,
lucenePort):
fieldRegistry = FieldRegistry(drilldownFields=drilldownFields)
luceneIndex = be((Lucene(
host='127.0.0.1',
port=lucenePort,
name=DEFAULT_CORE,
settings=defaultLuceneSettings.clone(fieldRegistry=fieldRegistry)),
(httpRequestAdapter, )))
return luceneIndex
def setUp(self, fieldRegistry=FieldRegistry()):
super(LuceneTestCase, self).setUp()
self._javaObjects = self._getJavaObjects()
self._reactor = CallTrace('reactor', methods={'addTimer': lambda seconds, callback: CallTrace('timer')})
self._defaultSettings = LuceneSettings(commitCount=1, commitTimeout=1, fieldRegistry=fieldRegistry)
self.lucene = Lucene(
join(self.tempdir, 'lucene'),
reactor=self._reactor,
settings=self._defaultSettings,
)
self.observer = CallTrace()
self.lucene.addObserver(self.observer)
def testScore(self):
reactor = CallTrace('reactor')
settings = LuceneSettings(commitCount=1, similarity=TermFrequencySimilarity(), verbose=False)
lucene = Lucene(join(self.tempdir, 'lucene'), reactor=reactor, settings=settings)
document = Document()
document.add(TextField('field', 'x '*100, Field.Store.NO))
returnValueFromGenerator(lucene.addDocument(identifier="identifier", document=document))
q = TermQuery(Term("field", 'x'))
result = returnValueFromGenerator(lucene.executeQuery(q))
self.assertAlmostEqual(0.1, result.hits[0].score)
q.setBoost(10.0)
result = returnValueFromGenerator(lucene.executeQuery(q))
self.assertAlmostEqual(1, result.hits[0].score)
def setUp(self):
SeecrTestCase.setUp(self)
self.registry = FieldRegistry()
self._multiLucene = MultiLucene(defaultCore='coreA', host="localhost", port=12345)
self._lucene = Lucene(host="localhost", port=12345, settings=LuceneSettings(), name='coreA')
self._multiLucene.addObserver(self._lucene)
self.post = []
self.response = ""
def mockPost(data, path, **kwargs):
self.post.append(dict(data=data, path=path))
raise StopIteration(self.response)
yield
connect = self._multiLucene._connect()
connect._post = mockPost
self._multiLucene._connect = lambda: connect
def testLuceneServerHostPortDynamic(self):
lucene = Lucene(name='lucene', settings=LuceneSettings(), readonly=True)
def httprequest1_1Mock(**kwargs):
raise StopIteration(parseResponse(HTTP_RESPONSE))
yield
observer = CallTrace(
'observer',
returnValues=dict(luceneServer=('example.org', 1234)),
methods=dict(httprequest1_1=httprequest1_1Mock))
lucene.addObserver(observer)
query = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
response = retval(lucene.executeQuery(
luceneQuery=query, start=1, stop=5,
))
self.assertEquals(887, response.total)
self.assertEquals(['luceneServer', 'httprequest1_1'], observer.calledMethodNames())
def setUpLucene(self, **kwargs):
self._lucene = Lucene(host="localhost", port=1234, name='lucene', settings=LuceneSettings(), **kwargs)
self.post = []
self.response = ""
connect = self._lucene._connect()
def mockPost(data, path, **kwargs):
self.post.append(dict(data=data, path=path))
raise StopIteration(self.response)
yield
connect._post = mockPost
self.read = []
self.response = ""
def mockRead(path, **kwargs):
self.read.append(path)
raise StopIteration(self.response)
yield
connect.read = mockRead
self._lucene._connect = lambda: connect
def main(reactor, port, databasePath):
drilldownFields = [
DrilldownField('untokenized.field2'),
DrilldownField('untokenized.fieldHier', hierarchical=True)
]
fieldRegistry = FieldRegistry(drilldownFields)
luceneSettings = LuceneSettings(fieldRegistry=fieldRegistry,
commitCount=30,
commitTimeout=1,
analyzer=MerescoDutchStemmingAnalyzer())
lucene = Lucene(path=join(databasePath, 'lucene'),
reactor=reactor,
name='main',
settings=luceneSettings)
lucene2Settings = LuceneSettings(fieldRegistry=fieldRegistry,
commitTimeout=0.1)
lucene2 = Lucene(path=join(databasePath, 'lucene2'),
reactor=reactor,
name='main2',
settings=lucene2Settings)
termNumerator = TermNumerator(path=join(databasePath, 'termNumerator'))
emptyLuceneSettings = LuceneSettings(commitTimeout=1)
multiLuceneHelix = (
MultiLucene(defaultCore='main'),
(Lucene(path=join(databasePath, 'lucene-empty'),
reactor=reactor,
name='empty-core',
settings=emptyLuceneSettings), ),
(lucene, ),
(lucene2, ),
)
storageComponent = StorageComponent(
directory=join(databasePath, 'storage'))
return \
(Observable(),
(ObservableHttpServer(reactor=reactor, port=port),
(BasicHttpHandler(),
(ApacheLogger(outputStream=stdout),
(PathFilter("/info", excluding=[
'/info/version',
'/info/name',
'/update',
'/sru',
'/remote',
'/via-remote-sru',
]),
(DynamicHtml(
[dynamicPath],
reactor=reactor,
indexPage='/info',
additionalGlobals={
'VERSION': version,
}
),
)
),
(PathFilter("/info/version"),
(StringServer(version, ContentTypePlainText), )
),
(PathFilter("/info/name"),
(StringServer('Meresco Lucene', ContentTypePlainText),)
),
(PathFilter("/static"),
(PathRename(lambda path: path[len('/static'):]),
(FileServer(staticPath),)
)
),
(PathFilter("/update_main", excluding=['/update_main2']),
uploadHelix(lucene, termNumerator, storageComponent, drilldownFields, fieldRegistry=luceneSettings.fieldRegistry),
),
(PathFilter("/update_main2"),
uploadHelix(lucene2, termNumerator, storageComponent, drilldownFields, fieldRegistry=lucene2Settings.fieldRegistry),
),
(PathFilter('/sru'),
(SruParser(defaultRecordSchema='record'),
(SruHandler(),
(MultiCqlToLuceneQuery(
defaultCore='main',
coreToCqlLuceneQueries={
"main": CqlToLuceneQuery([], luceneSettings=luceneSettings),
"main2": CqlToLuceneQuery([], luceneSettings=lucene2Settings),
"empty-core": CqlToLuceneQuery([], luceneSettings=emptyLuceneSettings),
}),
multiLuceneHelix,
),
(SRUTermDrilldown(defaultFormat='xml'),),
(SruDuplicateCount(),),
(storageComponent,),
)
)
),
(PathFilter('/via-remote-sru'),
(SruParser(defaultRecordSchema='record'),
(SruHandler(),
(LuceneRemote(host='localhost', port=port, path='/remote'),),
(SRUTermDrilldown(defaultFormat='xml'),),
(SruDuplicateCount(),),
(storageComponent,),
)
)
),
(PathFilter('/remote'),
(LuceneRemoteService(reactor=reactor),
(MultiCqlToLuceneQuery(
defaultCore='main',
coreToCqlLuceneQueries={
"main": CqlToLuceneQuery([], luceneSettings=luceneSettings),
"main2": CqlToLuceneQuery([], luceneSettings=lucene2Settings),
"empty-core": CqlToLuceneQuery([], luceneSettings=emptyLuceneSettings),
}),
multiLuceneHelix,
)
)
),
(PathFilter('/autocomplete'),
(Autocomplete('localhost', port, '/autocomplete', '__all__', '?', 5, '?', '?'),
(lucene,),
)
)
)
)
)
)
