def testUniteResultFromTwoIndexesCached(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[ dict(fieldname='cat_Q', maxTerms=10), dict(fieldname='cat_U', maxTerms=10), ]) q.setCoreQuery(core='coreB', query=None, facets=[ dict(fieldname='cat_N', maxTerms=10), dict(fieldname='cat_O', maxTerms=10), ]) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) resultOne = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('U=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=false')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) ignoredResult = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[ dict(fieldname='cat_Q', maxTerms=10), dict(fieldname='cat_U', maxTerms=10), ]) q.setCoreQuery(core='coreB', query=None, facets=[ dict(fieldname='cat_N', maxTerms=10), dict(fieldname='cat_O', maxTerms=10), ]) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) resultAgain = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(resultOne.total, resultAgain.total) self.assertEquals(resultOne.hits, resultAgain.hits) self.assertEquals(resultOne.drilldownData, resultAgain.drilldownData)
def testQueryOneIndexWithComposedQuery(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(cq)) self.assertEquals(set(['A-Q', 'A-QU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits)) cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), filterQueries=[luceneQueryFromCql('M=true')]) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(cq)) self.assertEquals(set(['A-MQ', 'A-MQU']), self.hitIds(result.hits))
def testUniteResultFromTwoIndexesCachedAfterUpdate(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[ dict(fieldname='cat_Q', maxTerms=10), dict(fieldname='cat_U', maxTerms=10), ]) q.setCoreQuery(core='coreB', query=None, facets=[ dict(fieldname='cat_N', maxTerms=10), dict(fieldname='cat_O', maxTerms=10), ]) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) resultOne = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(3, resultOne.total) self.assertEquals([{ 'terms': [ {'count': 3, 'term': u'true'} ], 'path': [], 'fieldname': u'cat_Q' }, { 'terms': [ {'count': 2, 'term': u'true'}, {'count': 1, 'term': u'false'} ], 'path': [], 'fieldname': u'cat_U' }, { 'terms': [ {'count': 2, 'term': u'true'} ], 'path': [], 'fieldname': u'cat_N' }, { 'terms': [ {'count': 1, 'term': u'true'}, {'count': 1, 'term': u'false'}, ], 'path': [], 'fieldname': u'cat_O' }], resultOne.drilldownData) self.addDocument(self.luceneA, identifier='A-MQU', keys=[('A', 8 )], fields=[('M', 'true' ), ('Q', 'false' ), ('U', 'true' ), ('S', '8')]) resultAgain = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(2, resultAgain.total) self.assertEquals([{ 'terms': [ {'count': 2, 'term': u'true'} ], 'path': [], 'fieldname': u'cat_Q' }, { 'terms': [ {'count': 1, 'term': u'false'}, {'count': 1, 'term': u'true'}, ], 'path': [], 'fieldname': u'cat_U' }, { 'terms': [ {'count': 1, 'term': u'true'} ], 'path': [], 'fieldname': u'cat_N' }, { 'terms': [ {'count': 1, 'term': u'true'} ], 'path': [], 'fieldname': u'cat_O' }], resultAgain.drilldownData)
def testCachingCollectorsAfterDelete(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=MatchAllDocsQuery()) q.setCoreQuery(core='coreB', query=luceneQueryFromCql("N=true")) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) self.addDocument(self.luceneB, identifier='B-N>A-MQU', keys=[('B', 8 )], fields=[('N', 'true' ), ('O', 'false'), ('P', 'false')]) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(set([u'A-M', u'A-MU', u'A-MQ', u'A-MQU']), self.hitIds(result.hits)) consume(self.luceneB.delete(identifier='B-N>A-MU')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(set([u'A-M', u'A-MQ', u'A-MQU']), self.hitIds(result.hits))
def testNullIteratorOfPForDeltaIsIgnoredInFinalKeySet(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('N=no_match')) q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'UNKOWN'), dict(core='coreB', key=KEY_PREFIX+'UNKOWN')) returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.luceneB.commit() # Force to write new segment; Old segment remains in seen list self.addDocument(self.luceneB, identifier='new', keys=[], fields=[('ignored', 'true')]) # Add new document to force recreating finalKeySet try: result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(0, len(result.hits)) finally: self.luceneB.delete(identifier='new')
def testScore(self): reactor = CallTrace('reactor') settings = LuceneSettings(commitCount=1, similarity=TermFrequencySimilarity(), verbose=False) lucene = Lucene(join(self.tempdir, 'lucene'), reactor=reactor, settings=settings) document = Document() document.add(TextField('field', 'x '*100, Field.Store.NO)) returnValueFromGenerator(lucene.addDocument(identifier="identifier", document=document)) q = TermQuery(Term("field", 'x')) result = returnValueFromGenerator(lucene.executeQuery(q)) self.assertAlmostEqual(0.1, result.hits[0].score) q.setBoost(10.0) result = returnValueFromGenerator(lucene.executeQuery(q)) self.assertAlmostEqual(1, result.hits[0].score)
def testJoinQueryWithFilters(self): q = ComposedQuery('coreA') q.addFilterQuery('coreB', query=luceneQueryFromCql('N=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(4, result.total) self.assertEquals(set(['A-M', 'A-MU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
def testRemoteExecuteQuery(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[ Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5") ]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost cq = ComposedQuery('coreA') cq.setCoreQuery( core='coreA', query=parseString('query AND field=value'), filterQueries=[parseString('query=fiets')], facets=[{ 'fieldname': 'field', 'maxTerms': 5 }], ) cq.setCoreQuery(core='coreB', query=parseString('query=test')) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) result = returnValueFromGenerator( observable.any.executeComposedQuery(query=cq)) self.assertEquals(5, result.total) self.assertEquals([ Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5") ], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) message, kwargs = jsonLoadMessage(m.kwargs['body']) query = kwargs['query'] self.assertEquals('executeComposedQuery', message) self.assertEquals('coreA', query.resultsFrom) self.assertEquals([{ 'fieldname': 'field', 'maxTerms': 5 }], query.facetsFor('coreA'))
def testRemotePrefixSearch(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse( total=5, hits=["1", "2", "3", "4", "5"]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator( observable.any.prefixSearch(prefix='aap', fieldname='field', limit=10)) self.assertEquals(5, result.total) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals( { 'message': 'prefixSearch', 'kwargs': { 'prefix': 'aap', 'fieldname': 'field', 'limit': 10, } }, loads(m.kwargs['body']))
def testJoinFacetAndQuery(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=None) q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'), facets=[ dict(fieldname='cat_N', maxTerms=10), dict(fieldname='cat_O', maxTerms=10), ]) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(query=q)) self.assertEquals(4, result.total) self.assertEquals(set(['A-M', 'A-MU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits)) self.assertEquals([{ 'terms': [ {'count': 4, 'term': u'true'}, ], 'path': [], 'fieldname': u'cat_N' }, { 'terms': [ {'count': 2, 'term': u'true'}, {'count': 2, 'term': u'false'}, ], 'path': [], 'fieldname': u'cat_O' }], result.drilldownData)
def testJoinQueryOnOptionalKeyOtherSide(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'D')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(1, result.total) self.assertEquals(set(['A-M']), self.hitIds(result.hits))
def testCachingKeyCollectorsIntersectsWithACopyOfTheKeys(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=MatchAllDocsQuery()) q.setCoreQuery(core='coreB', query=luceneQueryFromCql("O=true")) q.addFilterQuery(core='coreB', query=luceneQueryFromCql("N=true")) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(2, len(result.hits)) q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=MatchAllDocsQuery()) q.setCoreQuery(core='coreB', query=MatchAllDocsQuery()) q.addFilterQuery(core='coreB', query=luceneQueryFromCql("N=true")) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(4, len(result.hits))
def XXX_NOT_YET_IMPLEMENTED_testRankQueryInSingleCoreQuery(self): q = ComposedQuery('coreA', query=MatchAllDocsQuery()) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.setRankQuery(core='coreA', query=luceneQueryFromCql('Q=true')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(8, result.total) self.assertEquals([u'A-Q', u'A-QU', u'A-MQ', u'A-MQU', u'A', u'A-U', u'A-M', u'A-MU'], [hit.id for hit in result.hits])
def testRemoteExecuteQuery(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost cq = ComposedQuery('coreA') cq.setCoreQuery( core='coreA', query=parseString('query AND field=value'), filterQueries=[parseString('query=fiets')], facets=[{'fieldname': 'field', 'maxTerms':5}], ) cq.setCoreQuery(core='coreB', query=parseString('query=test')) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) result = returnValueFromGenerator(observable.any.executeComposedQuery(query=cq)) self.assertEquals(5, result.total) self.assertEquals([Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) message, kwargs = Conversion().jsonLoadMessage(m.kwargs['body']) query = kwargs['query'] self.assertEquals('executeComposedQuery', message) self.assertEquals('coreA', query.resultsFrom) self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], query.facetsFor('coreA'))
def testScoreCollectorCacheInvalidation(self): q = ComposedQuery('coreA', query=MatchAllDocsQuery()) q.setRankQuery(core='coreC', query=luceneQueryFromCql('S=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(8, result.total) self.assertEquals(u'A-MQU', result.hits[0].id) self.assertEquals(set([u'A', u'A-U', u'A-Q', u'A-QU', u'A-M', u'A-MU', u'A-MQ']), set([hit.id for hit in result.hits[1:]])) self.addDocument(self.luceneC, identifier='C-S>A-MQ', keys=[('C', 7)], fields=[('S', 'true')]) try: result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(8, result.total) self.assertEquals(set([u'A-MQ' , u'A-MQU']), set([hit.id for hit in result.hits[:2]])) self.assertEquals(set([u'A', u'A-U', u'A-Q', u'A-QU', u'A-M', u'A-MU']), set([hit.id for hit in result.hits[2:]])) finally: self.luceneC.delete(identifier='C-S>A-MQ')
def testMultipleJoinQueriesKeepsCachesWithinMaxSize(self): for i in xrange(25): self.addDocument(self.luceneB, identifier=str(i), keys=[('X', i)], fields=[('Y', str(i))]) for i in xrange(25): q = ComposedQuery('coreA', query=MatchAllDocsQuery()) q.setCoreQuery(core='coreB', query=luceneQueryFromCql('Y=%s' % i)) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'X')) ignoredResult = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
def testUniteMakesItTwoCoreQuery(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(3, result.total) self.assertEquals(set(['A-QU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
def testRankQuery(self): q = ComposedQuery('coreA', query=MatchAllDocsQuery()) q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true')) q.setRankQuery(core='coreC', query=luceneQueryFromCql('S=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(4, result.total) self.assertEquals([u'A-MQU', 'A-M', 'A-MU', u'A-MQ'], [hit.id for hit in result.hits])
def testKeyFilterIgnoresKeysOutOfBoundsOfKeySet(self): self.addDocument(self.luceneB, identifier=str(100), keys=[('B', 100)], fields=[]) # Force key to be much more than bits in long[] in FixedBitSet, so it must be OutOfBounds q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=MatchAllDocsQuery()) q.setCoreQuery(core='coreB', query=MatchAllDocsQuery()) q.addFacet(core='coreB', facet=dict(fieldname='cat_M', maxTerms=10)) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(4, len(result.hits))
def testRemoteExecuteQueryWithNoneValues(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse( total=5, hits=[Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator( observable.any.executeQuery( cqlAbstractSyntaxTree=parseString('query AND field=value'), start=0, stop=10, facets=None, filterQueries=None, joinQueries=None, )) self.assertEquals(5, result.total) self.assertEquals( [Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) self.assertDictEquals( { 'message': 'executeQuery', 'kwargs': { 'cqlAbstractSyntaxTree': { '__CQL_QUERY__': 'query AND field=value' }, 'start': 0, 'stop': 10, 'facets': None, 'filterQueries': None, 'joinQueries': None, } }, loads(m.kwargs['body']))
def testDeclineOtherMessages(self): class Other(object): def aMessage(self): raise StopIteration('Thanks') yield remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) observable.addObserver(Other()) result = returnValueFromGenerator(observable.any.aMessage()) self.assertEquals('Thanks', result)
def testCollectScoresWithNoResultAndBooleanQueryDoesntFailOnFakeScorerInAggregateScoreCollector(self): q = BooleanQuery() q.add(luceneQueryFromCql('M=true'), BooleanClause.Occur.SHOULD) q.add(luceneQueryFromCql('M=true'), BooleanClause.Occur.SHOULD) q = ComposedQuery('coreA', query=q) q.start = 0 q.stop = 0 q.setRankQuery(core='coreC', query=luceneQueryFromCql('S=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(4, result.total) self.assertEquals([], result.hits)
def testJoinFacetWithJoinDrilldownQueryFilters(self): q = ComposedQuery('coreA', query=luceneQueryFromCql('M=true')) q.addDrilldownQuery('coreB', drilldownQuery=('cat_O', ['true'])) q.addFacet('coreB', dict(fieldname='cat_O', maxTerms=10)) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(query=q)) self.assertEquals(2, result.total) self.assertEquals([{ 'terms': [ {'count': 3, 'term': u'true'}, ], 'path': [], 'fieldname': u'cat_O' }], result.drilldownData)
def testStartStopSortKeys(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true')) q.setCoreQuery(core='coreB', query=None) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) q.sortKeys=[dict(sortBy='S', sortDescending=False)] result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(3, result.total) self.assertEquals(set(['A-QU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits)) q.sortKeys=[dict(sortBy='S', sortDescending=True)] result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(3, result.total) self.assertEquals(set(['A-MQU', 'A-MQ', 'A-QU']), self.hitIds(result.hits)) q.stop = 2 result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(3, result.total) self.assertEquals(set(['A-MQU', 'A-MQ']), self.hitIds(result.hits)) q.stop = 10 q.start = 1 result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(3, result.total) self.assertEquals(set(['A-MQ', 'A-QU']), self.hitIds(result.hits))
def testJoinQueryThreeCores(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true')) q.setCoreQuery(core='coreC', query=luceneQueryFromCql('R=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C')) q.addFacet(core='coreA', facet=dict(fieldname='cat_M', maxTerms=10)) q.addFacet(core='coreB', facet=dict(fieldname='cat_N', maxTerms=10)) q.addFacet(core='coreC', facet=dict(fieldname='cat_R', maxTerms=10)) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(1, result.total) self.assertEquals(set(['A-M']), self.hitIds(result.hits)) self.assertEquals([ {'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_M'}, {'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_N'}, {'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_R'}, ], result.drilldownData)
def testJoinFacetWillNotFilter(self): query = ComposedQuery('coreA') query.setCoreQuery(core='coreA', query=None) query.setCoreQuery(core='coreB', query=None, facets=[ dict(fieldname='cat_N', maxTerms=10), ]) query.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(query=query)) self.assertEquals(8, result.total) self.assertEquals([{ 'terms': [ {'count': 4, 'term': u'true'}, {'count': 4, 'term': u'false'}, ], 'path': [], 'fieldname': u'cat_N' }], result.drilldownData)
def testUniteAndFacets(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[ dict(fieldname='cat_Q', maxTerms=10), dict(fieldname='cat_U', maxTerms=10), ]) q.setCoreQuery(core='coreB', query=None, facets=[ dict(fieldname='cat_N', maxTerms=10), dict(fieldname='cat_O', maxTerms=10), ]) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(3, result.total) self.assertEquals([{ 'terms': [ {'count': 3, 'term': u'true'}, ], 'path': [], 'fieldname': u'cat_Q' }, { 'terms': [ {'count': 2, 'term': u'true'}, {'count': 1, 'term': u'false'}, ], 'path': [], 'fieldname': u'cat_U' }, { 'terms': [ {'count': 2, 'term': u'true'}, ], 'path': [], 'fieldname': u'cat_N' }, { 'terms': [ {'count': 1, 'term': u'true'}, {'count': 1, 'term': u'false'}, ], 'path': [], 'fieldname': u'cat_O' }], result.drilldownData)
def testRemoteFieldnames(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=2, hits=["field0", "field1"]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator(observable.any.fieldnames()) self.assertEquals(2, result.total) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals({ 'message': 'fieldnames', 'kwargs':{ } }, loads(m.kwargs['body']))
def testRemoteExecuteQueryWithNoneValues(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator(observable.any.executeQuery( cqlAbstractSyntaxTree=parseString('query AND field=value'), start=0, stop=10, facets=None, filterQueries=None, joinQueries=None, ) ) self.assertEquals(5, result.total) self.assertEquals([Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) self.assertDictEquals({ 'message': 'executeQuery', 'kwargs':{ 'cqlAbstractSyntaxTree': {'__CQL_QUERY__': 'query AND field=value'}, 'start':0, 'stop': 10, 'facets': None, 'filterQueries': None, 'joinQueries': None, } }, loads(m.kwargs['body']))
def testRemoteFieldnames(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration( 'HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=2, hits=["field0", "field1"]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator(observable.any.fieldnames()) self.assertEquals(2, result.total) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals({ 'message': 'fieldnames', 'kwargs': {} }, loads(m.kwargs['body']))
def testRemotePrefixSearch(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=["1", "2", "3", "4", "5"]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost result = returnValueFromGenerator(observable.any.prefixSearch(prefix='aap', fieldname='field', limit=10)) self.assertEquals(5, result.total) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals({ 'message': 'prefixSearch', 'kwargs':{ 'prefix':'aap', 'fieldname': 'field', 'limit': 10, } }, loads(m.kwargs['body']))
def NOT_YET_SUPPORTED_testUniteAndFacetsResultsFromCoreB(self): q = ComposedQuery('coreB') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true')) q.setCoreQuery(core='coreB', query=None, facets=[ dict(fieldname='cat_N', maxTerms=10), dict(fieldname='cat_O', maxTerms=10), ]) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(set(['B-N>A-MQ', 'B-N>MQU']), self.hitIds(result.hits)) self.assertEquals(2, result.total) self.assertEquals([{ 'terms': [ {'count': 2, 'term': u'true'}, ], 'fieldname': u'cat_N' }, { 'terms': [ {'count': 1, 'term': u'false'}, {'count': 1, 'term': u'true'}, ], 'fieldname': u'cat_O' }], result.drilldownData)
def executeComposedQuery(self, *args, **kwargs): if len(args) == 1: kwargs['query'] = args[0] return returnValueFromGenerator( self._observable.any.unknown(message='executeComposedQuery', **kwargs))
def executeQuery(self, *args, **kwargs): if len(args) == 1: kwargs['cqlAbstractSyntaxTree'] = args[0] return returnValueFromGenerator( self._observable.any.unknown(message='executeQuery', **kwargs))
def drilldownFieldnames(self, **kwargs): return returnValueFromGenerator( self._observable.any.unknown(message='drilldownFieldnames', **kwargs))
def prefixSearch(self, **kwargs): return returnValueFromGenerator( self._observable.any.unknown(message='prefixSearch', **kwargs))
def NOT_YET_SUPPORTED_testUniteResultsFromCoreB(self): q = ComposedQuery('coreB') q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX +'A'), dict(core='coreB', key=KEY_PREFIX +'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(['B-N', 'B-N>A-M', 'B-N>A-MQ', 'B-N>A-MQU', 'B-N>A-MU', 'B-P>A-MQU', 'B-P>A-MU', ], sorted(result.hits))
def NOT_YET_SUPPORTED_testJoinQueryOnOptionalKeyUniteResultsWithoutKey(self): q = ComposedQuery('coreA') q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'C'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(set(['A-U', 'A-QU', 'A-MU', 'A-MQU', 'A-M']), self.hitIds(result.hits))