def testUniteResultFromTwoIndexesCached(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[
dict(fieldname='cat_Q', maxTerms=10),
dict(fieldname='cat_U', maxTerms=10),
])
q.setCoreQuery(core='coreB', query=None, facets=[
dict(fieldname='cat_N', maxTerms=10),
dict(fieldname='cat_O', maxTerms=10),
])
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
resultOne = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('U=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=false')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
ignoredResult = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[
dict(fieldname='cat_Q', maxTerms=10),
dict(fieldname='cat_U', maxTerms=10),
])
q.setCoreQuery(core='coreB', query=None, facets=[
dict(fieldname='cat_N', maxTerms=10),
dict(fieldname='cat_O', maxTerms=10),
])
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
resultAgain = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(resultOne.total, resultAgain.total)
self.assertEquals(resultOne.hits, resultAgain.hits)
self.assertEquals(resultOne.drilldownData, resultAgain.drilldownData)
def testQueryOneIndexWithComposedQuery(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(cq))
self.assertEquals(set(['A-Q', 'A-QU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), filterQueries=[luceneQueryFromCql('M=true')])
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(cq))
self.assertEquals(set(['A-MQ', 'A-MQU']), self.hitIds(result.hits))
def testUniteResultFromTwoIndexesCachedAfterUpdate(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[
dict(fieldname='cat_Q', maxTerms=10),
dict(fieldname='cat_U', maxTerms=10),
])
q.setCoreQuery(core='coreB', query=None, facets=[
dict(fieldname='cat_N', maxTerms=10),
dict(fieldname='cat_O', maxTerms=10),
])
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
resultOne = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(3, resultOne.total)
self.assertEquals([{
'terms': [
{'count': 3, 'term': u'true'}
], 'path': [], 'fieldname': u'cat_Q'
}, {
'terms': [
{'count': 2, 'term': u'true'},
{'count': 1, 'term': u'false'}
], 'path': [], 'fieldname': u'cat_U'
}, {
'terms': [
{'count': 2, 'term': u'true'}
], 'path': [], 'fieldname': u'cat_N'
}, {
'terms': [
{'count': 1, 'term': u'true'},
{'count': 1, 'term': u'false'},
], 'path': [], 'fieldname': u'cat_O'
}], resultOne.drilldownData)
self.addDocument(self.luceneA, identifier='A-MQU', keys=[('A', 8 )], fields=[('M', 'true' ), ('Q', 'false' ), ('U', 'true' ), ('S', '8')])
resultAgain = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(2, resultAgain.total)
self.assertEquals([{
'terms': [
{'count': 2, 'term': u'true'}
], 'path': [], 'fieldname': u'cat_Q'
}, {
'terms': [
{'count': 1, 'term': u'false'},
{'count': 1, 'term': u'true'},
], 'path': [], 'fieldname': u'cat_U'
}, {
'terms': [
{'count': 1, 'term': u'true'}
], 'path': [], 'fieldname': u'cat_N'
}, {
'terms': [
{'count': 1, 'term': u'true'}
], 'path': [], 'fieldname': u'cat_O'
}], resultAgain.drilldownData)
def testCachingCollectorsAfterDelete(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=MatchAllDocsQuery())
q.setCoreQuery(core='coreB', query=luceneQueryFromCql("N=true"))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
self.addDocument(self.luceneB, identifier='B-N>A-MQU', keys=[('B', 8 )], fields=[('N', 'true' ), ('O', 'false'), ('P', 'false')])
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(set([u'A-M', u'A-MU', u'A-MQ', u'A-MQU']), self.hitIds(result.hits))
consume(self.luceneB.delete(identifier='B-N>A-MU'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(set([u'A-M', u'A-MQ', u'A-MQU']), self.hitIds(result.hits))
def testNullIteratorOfPForDeltaIsIgnoredInFinalKeySet(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('N=no_match'))
q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'UNKOWN'), dict(core='coreB', key=KEY_PREFIX+'UNKOWN'))
returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.luceneB.commit() # Force to write new segment; Old segment remains in seen list
self.addDocument(self.luceneB, identifier='new', keys=[], fields=[('ignored', 'true')]) # Add new document to force recreating finalKeySet
try:
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(0, len(result.hits))
finally:
self.luceneB.delete(identifier='new')
def testScore(self):
reactor = CallTrace('reactor')
settings = LuceneSettings(commitCount=1, similarity=TermFrequencySimilarity(), verbose=False)
lucene = Lucene(join(self.tempdir, 'lucene'), reactor=reactor, settings=settings)
document = Document()
document.add(TextField('field', 'x '*100, Field.Store.NO))
returnValueFromGenerator(lucene.addDocument(identifier="identifier", document=document))
q = TermQuery(Term("field", 'x'))
result = returnValueFromGenerator(lucene.executeQuery(q))
self.assertAlmostEqual(0.1, result.hits[0].score)
q.setBoost(10.0)
result = returnValueFromGenerator(lucene.executeQuery(q))
self.assertAlmostEqual(1, result.hits[0].score)
def testJoinQueryWithFilters(self):
q = ComposedQuery('coreA')
q.addFilterQuery('coreB', query=luceneQueryFromCql('N=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(4, result.total)
self.assertEquals(set(['A-M', 'A-MU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
def testRemoteExecuteQuery(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' %
LuceneResponse(total=5,
hits=[
Hit("1"),
Hit("2"),
Hit("3", duplicateCount=2),
Hit("4"),
Hit("5")
]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
cq = ComposedQuery('coreA')
cq.setCoreQuery(
core='coreA',
query=parseString('query AND field=value'),
filterQueries=[parseString('query=fiets')],
facets=[{
'fieldname': 'field',
'maxTerms': 5
}],
)
cq.setCoreQuery(core='coreB', query=parseString('query=test'))
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
result = returnValueFromGenerator(
observable.any.executeComposedQuery(query=cq))
self.assertEquals(5, result.total)
self.assertEquals([
Hit("1"),
Hit("2"),
Hit("3", duplicateCount=2),
Hit("4"),
Hit("5")
], result.hits)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(1234, m.kwargs['port'])
self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
self.assertEquals('application/json',
m.kwargs['headers']['Content-Type'])
message, kwargs = jsonLoadMessage(m.kwargs['body'])
query = kwargs['query']
self.assertEquals('executeComposedQuery', message)
self.assertEquals('coreA', query.resultsFrom)
self.assertEquals([{
'fieldname': 'field',
'maxTerms': 5
}], query.facetsFor('coreA'))
def testRemotePrefixSearch(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(
total=5, hits=["1", "2", "3", "4", "5"]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
result = returnValueFromGenerator(
observable.any.prefixSearch(prefix='aap',
fieldname='field',
limit=10))
self.assertEquals(5, result.total)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(
{
'message': 'prefixSearch',
'kwargs': {
'prefix': 'aap',
'fieldname': 'field',
'limit': 10,
}
}, loads(m.kwargs['body']))
def testJoinFacetAndQuery(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=None)
q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'), facets=[
dict(fieldname='cat_N', maxTerms=10),
dict(fieldname='cat_O', maxTerms=10),
])
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(query=q))
self.assertEquals(4, result.total)
self.assertEquals(set(['A-M', 'A-MU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
self.assertEquals([{
'terms': [
{'count': 4, 'term': u'true'},
],
'path': [],
'fieldname': u'cat_N'
}, {
'terms': [
{'count': 2, 'term': u'true'},
{'count': 2, 'term': u'false'},
],
'path': [],
'fieldname': u'cat_O'
}], result.drilldownData)
def testJoinQueryOnOptionalKeyOtherSide(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'D'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(1, result.total)
self.assertEquals(set(['A-M']), self.hitIds(result.hits))
def testCachingKeyCollectorsIntersectsWithACopyOfTheKeys(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=MatchAllDocsQuery())
q.setCoreQuery(core='coreB', query=luceneQueryFromCql("O=true"))
q.addFilterQuery(core='coreB', query=luceneQueryFromCql("N=true"))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(2, len(result.hits))
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=MatchAllDocsQuery())
q.setCoreQuery(core='coreB', query=MatchAllDocsQuery())
q.addFilterQuery(core='coreB', query=luceneQueryFromCql("N=true"))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(4, len(result.hits))
def XXX_NOT_YET_IMPLEMENTED_testRankQueryInSingleCoreQuery(self):
q = ComposedQuery('coreA', query=MatchAllDocsQuery())
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.setRankQuery(core='coreA', query=luceneQueryFromCql('Q=true'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(8, result.total)
self.assertEquals([u'A-Q', u'A-QU', u'A-MQ', u'A-MQU', u'A', u'A-U', u'A-M', u'A-MU'], [hit.id for hit in result.hits])
def testRemoteExecuteQuery(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
cq = ComposedQuery('coreA')
cq.setCoreQuery(
core='coreA',
query=parseString('query AND field=value'),
filterQueries=[parseString('query=fiets')],
facets=[{'fieldname': 'field', 'maxTerms':5}],
)
cq.setCoreQuery(core='coreB', query=parseString('query=test'))
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
result = returnValueFromGenerator(observable.any.executeComposedQuery(query=cq))
self.assertEquals(5, result.total)
self.assertEquals([Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")], result.hits)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(1234, m.kwargs['port'])
self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
self.assertEquals('application/json', m.kwargs['headers']['Content-Type'])
message, kwargs = Conversion().jsonLoadMessage(m.kwargs['body'])
query = kwargs['query']
self.assertEquals('executeComposedQuery', message)
self.assertEquals('coreA', query.resultsFrom)
self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], query.facetsFor('coreA'))
def testScoreCollectorCacheInvalidation(self):
q = ComposedQuery('coreA', query=MatchAllDocsQuery())
q.setRankQuery(core='coreC', query=luceneQueryFromCql('S=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(8, result.total)
self.assertEquals(u'A-MQU', result.hits[0].id)
self.assertEquals(set([u'A', u'A-U', u'A-Q', u'A-QU', u'A-M', u'A-MU', u'A-MQ']), set([hit.id for hit in result.hits[1:]]))
self.addDocument(self.luceneC, identifier='C-S>A-MQ', keys=[('C', 7)], fields=[('S', 'true')])
try:
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(8, result.total)
self.assertEquals(set([u'A-MQ' , u'A-MQU']), set([hit.id for hit in result.hits[:2]]))
self.assertEquals(set([u'A', u'A-U', u'A-Q', u'A-QU', u'A-M', u'A-MU']), set([hit.id for hit in result.hits[2:]]))
finally:
self.luceneC.delete(identifier='C-S>A-MQ')
def testMultipleJoinQueriesKeepsCachesWithinMaxSize(self):
for i in xrange(25):
self.addDocument(self.luceneB, identifier=str(i), keys=[('X', i)], fields=[('Y', str(i))])
for i in xrange(25):
q = ComposedQuery('coreA', query=MatchAllDocsQuery())
q.setCoreQuery(core='coreB', query=luceneQueryFromCql('Y=%s' % i))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'X'))
ignoredResult = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
def testUniteMakesItTwoCoreQuery(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(3, result.total)
self.assertEquals(set(['A-QU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
def testRankQuery(self):
q = ComposedQuery('coreA', query=MatchAllDocsQuery())
q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'))
q.setRankQuery(core='coreC', query=luceneQueryFromCql('S=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(4, result.total)
self.assertEquals([u'A-MQU', 'A-M', 'A-MU', u'A-MQ'], [hit.id for hit in result.hits])
def testKeyFilterIgnoresKeysOutOfBoundsOfKeySet(self):
self.addDocument(self.luceneB, identifier=str(100), keys=[('B', 100)], fields=[]) # Force key to be much more than bits in long[] in FixedBitSet, so it must be OutOfBounds
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=MatchAllDocsQuery())
q.setCoreQuery(core='coreB', query=MatchAllDocsQuery())
q.addFacet(core='coreB', facet=dict(fieldname='cat_M', maxTerms=10))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(4, len(result.hits))
def testRemoteExecuteQueryWithNoneValues(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(
total=5,
hits=[Hit("1"),
Hit("2"),
Hit("3"),
Hit("4"),
Hit("5")]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
result = returnValueFromGenerator(
observable.any.executeQuery(
cqlAbstractSyntaxTree=parseString('query AND field=value'),
start=0,
stop=10,
facets=None,
filterQueries=None,
joinQueries=None,
))
self.assertEquals(5, result.total)
self.assertEquals(
[Hit("1"), Hit("2"),
Hit("3"), Hit("4"),
Hit("5")], result.hits)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(1234, m.kwargs['port'])
self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
self.assertEquals('application/json',
m.kwargs['headers']['Content-Type'])
self.assertDictEquals(
{
'message': 'executeQuery',
'kwargs': {
'cqlAbstractSyntaxTree': {
'__CQL_QUERY__': 'query AND field=value'
},
'start': 0,
'stop': 10,
'facets': None,
'filterQueries': None,
'joinQueries': None,
}
}, loads(m.kwargs['body']))
def testDeclineOtherMessages(self):
class Other(object):
def aMessage(self):
raise StopIteration('Thanks')
yield
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
observable.addObserver(Other())
result = returnValueFromGenerator(observable.any.aMessage())
self.assertEquals('Thanks', result)
def testCollectScoresWithNoResultAndBooleanQueryDoesntFailOnFakeScorerInAggregateScoreCollector(self):
q = BooleanQuery()
q.add(luceneQueryFromCql('M=true'), BooleanClause.Occur.SHOULD)
q.add(luceneQueryFromCql('M=true'), BooleanClause.Occur.SHOULD)
q = ComposedQuery('coreA', query=q)
q.start = 0
q.stop = 0
q.setRankQuery(core='coreC', query=luceneQueryFromCql('S=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(4, result.total)
self.assertEquals([], result.hits)
def testDeclineOtherMessages(self):
class Other(object):
def aMessage(self):
raise StopIteration('Thanks')
yield
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
observable.addObserver(Other())
result = returnValueFromGenerator(observable.any.aMessage())
self.assertEquals('Thanks', result)
def testJoinFacetWithJoinDrilldownQueryFilters(self):
q = ComposedQuery('coreA', query=luceneQueryFromCql('M=true'))
q.addDrilldownQuery('coreB', drilldownQuery=('cat_O', ['true']))
q.addFacet('coreB', dict(fieldname='cat_O', maxTerms=10))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(query=q))
self.assertEquals(2, result.total)
self.assertEquals([{
'terms': [
{'count': 3, 'term': u'true'},
],
'path': [],
'fieldname': u'cat_O'
}], result.drilldownData)
def testStartStopSortKeys(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'))
q.setCoreQuery(core='coreB', query=None)
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
q.sortKeys=[dict(sortBy='S', sortDescending=False)]
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(3, result.total)
self.assertEquals(set(['A-QU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
q.sortKeys=[dict(sortBy='S', sortDescending=True)]
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(3, result.total)
self.assertEquals(set(['A-MQU', 'A-MQ', 'A-QU']), self.hitIds(result.hits))
q.stop = 2
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(3, result.total)
self.assertEquals(set(['A-MQU', 'A-MQ']), self.hitIds(result.hits))
q.stop = 10
q.start = 1
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(3, result.total)
self.assertEquals(set(['A-MQ', 'A-QU']), self.hitIds(result.hits))
def testJoinQueryThreeCores(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'))
q.setCoreQuery(core='coreC', query=luceneQueryFromCql('R=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C'))
q.addFacet(core='coreA', facet=dict(fieldname='cat_M', maxTerms=10))
q.addFacet(core='coreB', facet=dict(fieldname='cat_N', maxTerms=10))
q.addFacet(core='coreC', facet=dict(fieldname='cat_R', maxTerms=10))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(1, result.total)
self.assertEquals(set(['A-M']), self.hitIds(result.hits))
self.assertEquals([
{'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_M'},
{'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_N'},
{'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_R'},
], result.drilldownData)
def testJoinFacetWillNotFilter(self):
query = ComposedQuery('coreA')
query.setCoreQuery(core='coreA', query=None)
query.setCoreQuery(core='coreB', query=None, facets=[
dict(fieldname='cat_N', maxTerms=10),
])
query.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(query=query))
self.assertEquals(8, result.total)
self.assertEquals([{
'terms': [
{'count': 4, 'term': u'true'},
{'count': 4, 'term': u'false'},
],
'path': [],
'fieldname': u'cat_N'
}], result.drilldownData)
def testUniteAndFacets(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[
dict(fieldname='cat_Q', maxTerms=10),
dict(fieldname='cat_U', maxTerms=10),
])
q.setCoreQuery(core='coreB', query=None, facets=[
dict(fieldname='cat_N', maxTerms=10),
dict(fieldname='cat_O', maxTerms=10),
])
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(3, result.total)
self.assertEquals([{
'terms': [
{'count': 3, 'term': u'true'},
],
'path': [],
'fieldname': u'cat_Q'
}, {
'terms': [
{'count': 2, 'term': u'true'},
{'count': 1, 'term': u'false'},
],
'path': [],
'fieldname': u'cat_U'
}, {
'terms': [
{'count': 2, 'term': u'true'},
],
'path': [],
'fieldname': u'cat_N'
}, {
'terms': [
{'count': 1, 'term': u'true'},
{'count': 1, 'term': u'false'},
],
'path': [],
'fieldname': u'cat_O'
}], result.drilldownData)
def testRemoteFieldnames(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=2, hits=["field0", "field1"]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
result = returnValueFromGenerator(observable.any.fieldnames())
self.assertEquals(2, result.total)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals({
'message': 'fieldnames',
'kwargs':{
}
}, loads(m.kwargs['body']))
def testRemoteExecuteQueryWithNoneValues(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
result = returnValueFromGenerator(observable.any.executeQuery(
cqlAbstractSyntaxTree=parseString('query AND field=value'),
start=0,
stop=10,
facets=None,
filterQueries=None,
joinQueries=None,
)
)
self.assertEquals(5, result.total)
self.assertEquals([Hit("1"), Hit("2"), Hit("3"), Hit("4"), Hit("5")], result.hits)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(1234, m.kwargs['port'])
self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
self.assertEquals('application/json', m.kwargs['headers']['Content-Type'])
self.assertDictEquals({
'message': 'executeQuery',
'kwargs':{
'cqlAbstractSyntaxTree': {'__CQL_QUERY__': 'query AND field=value'},
'start':0,
'stop': 10,
'facets': None,
'filterQueries': None,
'joinQueries': None,
}
}, loads(m.kwargs['body']))
def testRemoteFieldnames(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration(
'HTTP/1.0 200 Ok\r\n\r\n%s' %
LuceneResponse(total=2, hits=["field0", "field1"]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
result = returnValueFromGenerator(observable.any.fieldnames())
self.assertEquals(2, result.total)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals({
'message': 'fieldnames',
'kwargs': {}
}, loads(m.kwargs['body']))
def testRemotePrefixSearch(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=["1", "2", "3", "4", "5"]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
result = returnValueFromGenerator(observable.any.prefixSearch(prefix='aap', fieldname='field', limit=10))
self.assertEquals(5, result.total)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals({
'message': 'prefixSearch',
'kwargs':{
'prefix':'aap',
'fieldname': 'field',
'limit': 10,
}
}, loads(m.kwargs['body']))
def NOT_YET_SUPPORTED_testUniteAndFacetsResultsFromCoreB(self):
q = ComposedQuery('coreB')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'))
q.setCoreQuery(core='coreB', query=None, facets=[
dict(fieldname='cat_N', maxTerms=10),
dict(fieldname='cat_O', maxTerms=10),
])
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(set(['B-N>A-MQ', 'B-N>MQU']), self.hitIds(result.hits))
self.assertEquals(2, result.total)
self.assertEquals([{
'terms': [
{'count': 2, 'term': u'true'},
],
'fieldname': u'cat_N'
}, {
'terms': [
{'count': 1, 'term': u'false'},
{'count': 1, 'term': u'true'},
],
'fieldname': u'cat_O'
}], result.drilldownData)
def executeComposedQuery(self, *args, **kwargs):
if len(args) == 1:
kwargs['query'] = args[0]
return returnValueFromGenerator(
self._observable.any.unknown(message='executeComposedQuery',
**kwargs))
def executeQuery(self, *args, **kwargs):
if len(args) == 1:
kwargs['cqlAbstractSyntaxTree'] = args[0]
return returnValueFromGenerator(
self._observable.any.unknown(message='executeQuery', **kwargs))
def drilldownFieldnames(self, **kwargs):
return returnValueFromGenerator(
self._observable.any.unknown(message='drilldownFieldnames',
**kwargs))
def prefixSearch(self, **kwargs):
return returnValueFromGenerator(
self._observable.any.unknown(message='prefixSearch', **kwargs))
def NOT_YET_SUPPORTED_testUniteResultsFromCoreB(self):
q = ComposedQuery('coreB')
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX +'A'), dict(core='coreB', key=KEY_PREFIX +'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(['B-N', 'B-N>A-M', 'B-N>A-MQ', 'B-N>A-MQU', 'B-N>A-MU', 'B-P>A-MQU', 'B-P>A-MU', ], sorted(result.hits))
def NOT_YET_SUPPORTED_testJoinQueryOnOptionalKeyUniteResultsWithoutKey(self):
q = ComposedQuery('coreA')
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'C'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(set(['A-U', 'A-QU', 'A-MU', 'A-MQU', 'A-M']), self.hitIds(result.hits))