def testRemoteExecuteQuery(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[ Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5") ]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost cq = ComposedQuery('coreA') cq.setCoreQuery( core='coreA', query=parseString('query AND field=value'), filterQueries=[parseString('query=fiets')], facets=[{ 'fieldname': 'field', 'maxTerms': 5 }], ) cq.setCoreQuery(core='coreB', query=parseString('query=test')) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) result = returnValueFromGenerator( observable.any.executeComposedQuery(query=cq)) self.assertEquals(5, result.total) self.assertEquals([ Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5") ], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) message, kwargs = jsonLoadMessage(m.kwargs['body']) query = kwargs['query'] self.assertEquals('executeComposedQuery', message) self.assertEquals('coreA', query.resultsFrom) self.assertEquals([{ 'fieldname': 'field', 'maxTerms': 5 }], query.facetsFor('coreA'))
def testRemoteExecuteQuery(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost cq = ComposedQuery('coreA') cq.setCoreQuery( core='coreA', query=parseString('query AND field=value'), filterQueries=[parseString('query=fiets')], facets=[{'fieldname': 'field', 'maxTerms':5}], ) cq.setCoreQuery(core='coreB', query=parseString('query=test')) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) result = returnValueFromGenerator(observable.any.executeComposedQuery(query=cq)) self.assertEquals(5, result.total) self.assertEquals([Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) message, kwargs = Conversion().jsonLoadMessage(m.kwargs['body']) query = kwargs['query'] self.assertEquals('executeComposedQuery', message) self.assertEquals('coreA', query.resultsFrom) self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], query.facetsFor('coreA'))
def testRemotePrefixSearch(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse( total=5, hits=["1", "2", "3", "4", "5"]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator( observable.any.prefixSearch(prefix='aap', fieldname='field', limit=10)) self.assertEquals(5, result.total) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals( { 'message': 'prefixSearch', 'kwargs': { 'prefix': 'aap', 'fieldname': 'field', 'limit': 10, } }, loads(m.kwargs['body']))
def testRemoteExecuteQueryWithNoneValues(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse( total=5, hits=[Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator( observable.any.executeQuery( cqlAbstractSyntaxTree=parseString('query AND field=value'), start=0, stop=10, facets=None, filterQueries=None, joinQueries=None, )) self.assertEquals(5, result.total) self.assertEquals( [Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) self.assertDictEquals( { 'message': 'executeQuery', 'kwargs': { 'cqlAbstractSyntaxTree': { '__CQL_QUERY__': 'query AND field=value' }, 'start': 0, 'stop': 10, 'facets': None, 'filterQueries': None, 'joinQueries': None, } }, loads(m.kwargs['body']))
def testRemoteFieldnames(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=2, hits=["field0", "field1"]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator(observable.any.fieldnames()) self.assertEquals(2, result.total) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals({ 'message': 'fieldnames', 'kwargs':{ } }, loads(m.kwargs['body']))
def testDeclineOtherMessages(self): class Other(object): def aMessage(self): raise StopIteration('Thanks') yield remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) observable.addObserver(Other()) result = returnValueFromGenerator(observable.any.aMessage()) self.assertEquals('Thanks', result)
def testRemoteExecuteQueryWithNoneValues(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator(observable.any.executeQuery( cqlAbstractSyntaxTree=parseString('query AND field=value'), start=0, stop=10, facets=None, filterQueries=None, joinQueries=None, ) ) self.assertEquals(5, result.total) self.assertEquals([Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) self.assertDictEquals({ 'message': 'executeQuery', 'kwargs':{ 'cqlAbstractSyntaxTree': {'__CQL_QUERY__': 'query AND field=value'}, 'start':0, 'stop': 10, 'facets': None, 'filterQueries': None, 'joinQueries': None, } }, loads(m.kwargs['body']))
def testRemotePrefixSearch(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=["1", "2", "3", "4", "5"]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator(observable.any.prefixSearch(prefix='aap', fieldname='field', limit=10)) self.assertEquals(5, result.total) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals({ 'message': 'prefixSearch', 'kwargs':{ 'prefix':'aap', 'fieldname': 'field', 'limit': 10, } }, loads(m.kwargs['body']))
def testRemoteFieldnames(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration( 'HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=2, hits=["field0", "field1"]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator(observable.any.fieldnames()) self.assertEquals(2, result.total) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals({ 'message': 'fieldnames', 'kwargs': {} }, loads(m.kwargs['body']))
def main(reactor, port, databasePath): drilldownFields = [ DrilldownField('untokenized.field2'), DrilldownField('untokenized.fieldHier', hierarchical=True) ] fieldRegistry = FieldRegistry(drilldownFields) luceneSettings = LuceneSettings(fieldRegistry=fieldRegistry, commitCount=30, commitTimeout=1, analyzer=MerescoDutchStemmingAnalyzer()) lucene = Lucene(path=join(databasePath, 'lucene'), reactor=reactor, name='main', settings=luceneSettings) lucene2Settings = LuceneSettings(fieldRegistry=fieldRegistry, commitTimeout=0.1) lucene2 = Lucene(path=join(databasePath, 'lucene2'), reactor=reactor, name='main2', settings=lucene2Settings) termNumerator = TermNumerator(path=join(databasePath, 'termNumerator')) emptyLuceneSettings = LuceneSettings(commitTimeout=1) multiLuceneHelix = ( MultiLucene(defaultCore='main'), (Lucene(path=join(databasePath, 'lucene-empty'), reactor=reactor, name='empty-core', settings=emptyLuceneSettings), ), (lucene, ), (lucene2, ), ) storageComponent = StorageComponent( directory=join(databasePath, 'storage')) return \ (Observable(), (ObservableHttpServer(reactor=reactor, port=port), (BasicHttpHandler(), (ApacheLogger(outputStream=stdout), (PathFilter("/info", excluding=[ '/info/version', '/info/name', '/update', '/sru', '/remote', '/via-remote-sru', ]), (DynamicHtml( [dynamicPath], reactor=reactor, indexPage='/info', additionalGlobals={ 'VERSION': version, } ), ) ), (PathFilter("/info/version"), (StringServer(version, ContentTypePlainText), ) ), (PathFilter("/info/name"), (StringServer('Meresco Lucene', ContentTypePlainText),) ), (PathFilter("/static"), (PathRename(lambda path: path[len('/static'):]), (FileServer(staticPath),) ) ), (PathFilter("/update_main", excluding=['/update_main2']), uploadHelix(lucene, termNumerator, storageComponent, drilldownFields, fieldRegistry=luceneSettings.fieldRegistry), ), (PathFilter("/update_main2"), uploadHelix(lucene2, termNumerator, storageComponent, drilldownFields, fieldRegistry=lucene2Settings.fieldRegistry), ), (PathFilter('/sru'), (SruParser(defaultRecordSchema='record'), (SruHandler(), (MultiCqlToLuceneQuery( defaultCore='main', coreToCqlLuceneQueries={ "main": CqlToLuceneQuery([], luceneSettings=luceneSettings), "main2": CqlToLuceneQuery([], luceneSettings=lucene2Settings), "empty-core": CqlToLuceneQuery([], luceneSettings=emptyLuceneSettings), }), multiLuceneHelix, ), (SRUTermDrilldown(defaultFormat='xml'),), (SruDuplicateCount(),), (storageComponent,), ) ) ), (PathFilter('/via-remote-sru'), (SruParser(defaultRecordSchema='record'), (SruHandler(), (LuceneRemote(host='localhost', port=port, path='/remote'),), (SRUTermDrilldown(defaultFormat='xml'),), (SruDuplicateCount(),), (storageComponent,), ) ) ), (PathFilter('/remote'), (LuceneRemoteService(reactor=reactor), (MultiCqlToLuceneQuery( defaultCore='main', coreToCqlLuceneQueries={ "main": CqlToLuceneQuery([], luceneSettings=luceneSettings), "main2": CqlToLuceneQuery([], luceneSettings=lucene2Settings), "empty-core": CqlToLuceneQuery([], luceneSettings=emptyLuceneSettings), }), multiLuceneHelix, ) ) ), (PathFilter('/autocomplete'), (Autocomplete('localhost', port, '/autocomplete', '__all__', '?', 5, '?', '?'), (lucene,), ) ) ) ) ) )