def testInfoOnQuery(self):
self.response = JsonDict({
"total": 887,
"queryTime": 6,
"hits": [{"id": "record:1", "score": 0.1234}]
}).dumps()
q = ComposedQuery('coreA')
q.addFilterQuery('coreB', query='N=true')
q.addMatch(dict(core='coreA', uniqueKey='A'), dict(core='coreB', key='B'))
result = retval(self._multiLucene.executeComposedQuery(q))
self.assertEquals({
'query': {
'cores': ['coreB', 'coreA'],
'drilldownQueries': {},
'facets': {},
'filterQueries': {'coreB': ['N=true']},
'matches': {'coreA->coreB': [{'core': 'coreA', 'uniqueKey': 'A'}, {'core': 'coreB', 'key': 'B'}]},
'otherCoreFacetFilters': {},
'queries': {},
'rankQueries': {},
'resultsFrom': 'coreA',
'sortKeys': [],
'unites': []
},
'type': 'ComposedQuery'
}, result.info)
def testRemoteExecuteQuery(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' %
LuceneResponse(total=5,
hits=[
Hit("1"),
Hit("2"),
Hit("3", duplicateCount=2),
Hit("4"),
Hit("5")
]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
cq = ComposedQuery('coreA')
cq.setCoreQuery(
core='coreA',
query=parseString('query AND field=value'),
filterQueries=[parseString('query=fiets')],
facets=[{
'fieldname': 'field',
'maxTerms': 5
}],
)
cq.setCoreQuery(core='coreB', query=parseString('query=test'))
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
result = returnValueFromGenerator(
observable.any.executeComposedQuery(query=cq))
self.assertEquals(5, result.total)
self.assertEquals([
Hit("1"),
Hit("2"),
Hit("3", duplicateCount=2),
Hit("4"),
Hit("5")
], result.hits)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(1234, m.kwargs['port'])
self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
self.assertEquals('application/json',
m.kwargs['headers']['Content-Type'])
message, kwargs = jsonLoadMessage(m.kwargs['body'])
query = kwargs['query']
self.assertEquals('executeComposedQuery', message)
self.assertEquals('coreA', query.resultsFrom)
self.assertEquals([{
'fieldname': 'field',
'maxTerms': 5
}], query.facetsFor('coreA'))
def testAddTypeAndMissingValueToSortField(self):
self.response = JsonDict({"total": 887, "queryTime": 6, "hits": [{"id": "record:1", "score": 0.1234}]}).dumps()
cq = ComposedQuery("coreA")
q = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
cq.setCoreQuery("coreB", q)
cq.sortKeys = [dict(sortBy="sortField", core="coreA", sortDescending=True)]
cq.addMatch(dict(core="coreA", uniqueKey="A"), dict(core="coreB", key="B"))
consume(self._multiLucene.executeComposedQuery(cq))
self.assertEqual(
{
"_sortKeys": [
{
"core": "coreA",
"sortBy": "sortField",
"sortDescending": True,
"type": "String",
"missingValue": "STRING_FIRST",
}
],
"resultsFrom": "coreA",
"_matches": {"coreA->coreB": [{"core": "coreA", "uniqueKey": "A"}, {"core": "coreB", "key": "B"}]},
"_facets": {},
"_otherCoreFacetFilters": {},
"_rankQueries": {},
"_drilldownQueries": {},
"_unites": [],
"_queries": {"coreB": {"term": {"field": "field", "value": "value"}, "type": "TermQuery"}},
"cores": ["coreB", "coreA"],
"_filterQueries": {},
},
loads(self.post[0]["data"]),
)
def testConvertAllQueriesWithUnqualifiedTermFields(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query='Q0')
cq.addFilterQuery('coreA', 'Q1')
cq.addFilterQuery('coreA', 'Q2')
cq.setCoreQuery(core='coreB', query='Q3')
cq.addFilterQuery('coreB', 'Q4')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='Q5'),
dict(core='coreB', query='Q6'))
cq.unqualifiedTermFields = [('field0', 2.0), ('field1', 3.0)]
convertCoreA = lambda query, unqualifiedTermFields=None, **kwargs: "Converted_A_{0}_{1}".format(
query, not unqualifiedTermFields is None)
convertCoreB = lambda query, **kwargs: "Converted_B_{0}".format(query)
cq.convertWith(coreA=convertCoreA, coreB=convertCoreB)
self.assertEquals("Converted_A_Q0_True", cq.queryFor('coreA'))
self.assertEquals(["Converted_A_Q1_True", "Converted_A_Q2_True"],
cq.filterQueriesFor('coreA'))
self.assertEquals("Converted_B_Q3", cq.queryFor('coreB'))
self.assertEquals(["Converted_B_Q4"], cq.filterQueriesFor('coreB'))
self.assertEqual(1, len(cq.unites))
uniteQueries = list(cq.unites[0].queries())
self.assertEquals('Converted_A_Q5_True', uniteQueries[0][0]['query'])
self.assertEquals('Converted_B_Q6', uniteQueries[1][0]['query'])
def testUnite(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query=None)
cq.setCoreQuery(core='coreB', query=None)
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery'))
self.assertEquals([('coreA', 'AQuery'), ('coreB', 'anotherQuery')], cq.unites)
def testKeyName(self):
composedQuery = ComposedQuery('coreA')
composedQuery.setCoreQuery(core='coreA', query=None)
composedQuery.setCoreQuery(core='coreB', query=None)
composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
self.assertEquals('keyA', composedQuery.keyName('coreA'))
self.assertEquals('keyB', composedQuery.keyName('coreB'))
def testFilterQueries(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query='Q0')
cq.addFilterQuery(core='coreA', query='Q1')
cq.addFilterQuery(core='coreA', query='Q2')
cq.addFacet(core='coreA', facet='F0')
cq.addFacet(core='coreA', facet='F1')
cq.setCoreQuery(core='coreB', query='Q3')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='AQuery'),
dict(core='coreB', query='anotherQuery'))
self.assertEquals(None, cq.stop)
self.assertEquals(None, cq.start)
self.assertEquals([], cq.sortKeys)
cq.stop = 10
cq.start = 0
cq.sortKeys = [dict(sortBy='field', sortDescending=True)]
self.assertEquals('Q0', cq.queryFor('coreA'))
self.assertEquals(['Q1', 'Q2'], cq.filterQueriesFor('coreA'))
self.assertEquals(['F0', 'F1'], cq.facetsFor('coreA'))
self.assertEquals(10, cq.stop)
self.assertEquals(0, cq.start)
self.assertEquals([dict(sortBy='field', sortDescending=True)],
cq.sortKeys)
def testJoinQueryWithFilters(self):
q = ComposedQuery('coreA')
q.addFilterQuery('coreB', query=luceneQueryFromCql('N=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(4, result.total)
self.assertEquals(set(['A-M', 'A-MU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
def testKeyName(self):
cq = ComposedQuery('coreA')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.addFacet(core='coreB', facet='F0')
self.assertEquals('keyA', cq.keyName('coreA', 'coreB'))
self.assertEquals('keyB', cq.keyName('coreB', 'coreA'))
self.assertEquals(set(['keyA']), cq.keyNames('coreA'))
def testIsSingleCoreQuery(self):
cq = ComposedQuery('coreA')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.setCoreQuery('coreA', query='A')
self.assertTrue(cq.isSingleCoreQuery())
cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
self.assertFalse(cq.isSingleCoreQuery())
def testIsSingleCoreQuery(self):
cq = ComposedQuery('coreA')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.setCoreQuery('coreA', query='A')
self.assertTrue(cq.isSingleCoreQuery())
cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
self.assertFalse(cq.isSingleCoreQuery())
def XXX_NOT_YET_IMPLEMENTED_testRankQueryInSingleCoreQuery(self):
q = ComposedQuery('coreA', query=MatchAllDocsQuery())
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.setRankQuery(core='coreA', query=luceneQueryFromCql('Q=true'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(8, result.total)
self.assertEquals([u'A-Q', u'A-QU', u'A-MQ', u'A-MQU', u'A', u'A-U', u'A-M', u'A-MU'], [hit.id for hit in result.hits])
def testJoinQueryOnOptionalKeyOtherSide(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'D'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(1, result.total)
self.assertEquals(set(['A-M']), self.hitIds(result.hits))
def testAsDictFromDict(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query='Q0')
cq.addFilterQuery(core='coreA', query='Q1')
cq.addFilterQuery(core='coreA', query='Q2')
cq.addFacet(core='coreA', facet='F0')
cq.addFacet(core='coreA', facet='F1')
cq.setCoreQuery(core='coreB', query='Q3')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery'))
cq.start = 0
cq.sortKeys = [dict(sortBy='field', sortDescending=True)]
cq.clustering = True
cq.clusteringConfig = {'clusteringEps': 0.2}
d = cq.asDict()
cq2 = ComposedQuery.fromDict(d)
self.assertEquals('coreA', cq2.resultsFrom)
self.assertEquals(0, cq2.start)
self.assertEquals(None, cq2.stop)
self.assertEquals(['Q0', 'Q1', 'Q2'], cq2.queriesFor('coreA'))
self.assertEquals(['F0', 'F1'], cq2.facetsFor('coreA'))
self.assertEquals('keyA', cq2.keyName('coreA', 'coreB'))
self.assertEquals('keyB', cq2.keyName('coreB', 'coreA'))
self.assertEqual(1, len(cq2.unites))
queries = list(cq2.unites[0].queries())
self.assertEquals(({'core': 'coreA', 'keyName': 'keyA', 'query': 'AQuery'}, 'keyA'), queries[0])
self.assertEquals(({'core': 'coreB', 'keyName': 'keyB', 'query': 'anotherQuery'}, 'keyA'), queries[1])
self.assertEquals({'clusteringEps': 0.2}, cq2.clusteringConfig)
def testJoinFacetAndQuery(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=None)
q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'), facets=[
dict(fieldname='cat_N', maxTerms=10),
dict(fieldname='cat_O', maxTerms=10),
])
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(query=q))
self.assertEquals(4, result.total)
self.assertEquals(set(['A-M', 'A-MU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
self.assertEquals([{
'terms': [
{'count': 4, 'term': u'true'},
],
'path': [],
'fieldname': u'cat_N'
}, {
'terms': [
{'count': 2, 'term': u'true'},
{'count': 2, 'term': u'false'},
],
'path': [],
'fieldname': u'cat_O'
}], result.drilldownData)
def testUniteMakesItTwoCoreQuery(self):
cq = ComposedQuery('coreA')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.setCoreQuery('coreA', query='A')
cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
cq.validate()
self.assertEquals(set(['coreA', 'coreB']), cq.cores)
def testUniteMakesItTwoCoreQuery(self):
cq = ComposedQuery('coreA')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.setCoreQuery('coreA', query='A')
cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
cq.validate()
self.assertEquals(set(['coreA', 'coreB']), cq.cores)
def testAddTypeAndMissingValueToSortField(self):
self.response = JsonDict({
"total": 887,
"queryTime": 6,
"hits": [{"id": "record:1", "score": 0.1234}]
}).dumps()
cq = ComposedQuery('coreA')
q = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
cq.setCoreQuery('coreB', q)
cq.sortKeys = [dict(sortBy='sortField', core='coreA', sortDescending=True)]
cq.addMatch(dict(core='coreA', uniqueKey='A'), dict(core='coreB', key='B'))
consume(self._multiLucene.executeComposedQuery(cq))
self.assertEqual({
"_sortKeys": [{'core': 'coreA', 'sortBy': 'sortField', 'sortDescending': True, 'type': 'String', 'missingValue': 'STRING_FIRST'}],
"resultsFrom": "coreA",
'_matches': {'coreA->coreB': [{'core': 'coreA', 'uniqueKey': 'A'}, {'core': 'coreB', 'key': 'B'}]},
"_facets": {},
"_otherCoreFacetFilters": {},
"_rankQueries": {},
"_drilldownQueries": {},
"_unites": [],
'_queries': {'coreB': {'term': {'field': 'field', 'value': 'value'}, 'type': 'TermQuery'}},
"cores": ["coreB", "coreA"],
"_filterQueries": {}
}, loads(self.post[0]['data']))
def testRemoteExecuteQuery(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
cq = ComposedQuery('coreA')
cq.setCoreQuery(
core='coreA',
query=parseString('query AND field=value'),
filterQueries=[parseString('query=fiets')],
facets=[{'fieldname': 'field', 'maxTerms':5}],
)
cq.setCoreQuery(core='coreB', query=parseString('query=test'))
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
result = returnValueFromGenerator(observable.any.executeComposedQuery(query=cq))
self.assertEquals(5, result.total)
self.assertEquals([Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")], result.hits)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(1234, m.kwargs['port'])
self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
self.assertEquals('application/json', m.kwargs['headers']['Content-Type'])
message, kwargs = Conversion().jsonLoadMessage(m.kwargs['body'])
query = kwargs['query']
self.assertEquals('executeComposedQuery', message)
self.assertEquals('coreA', query.resultsFrom)
self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], query.facetsFor('coreA'))
def testInfoOnQuery(self):
self.response = JsonDict({"total": 887, "queryTime": 6, "hits": [{"id": "record:1", "score": 0.1234}]}).dumps()
q = ComposedQuery("coreA")
q.addFilterQuery("coreB", query="N=true")
q.addMatch(dict(core="coreA", uniqueKey="A"), dict(core="coreB", key="B"))
result = retval(self._multiLucene.executeComposedQuery(q))
self.assertEquals(
{
"query": {
"cores": ["coreB", "coreA"],
"drilldownQueries": {},
"facets": {},
"filterQueries": {"coreB": ["N=true"]},
"matches": {"coreA->coreB": [{"core": "coreA", "uniqueKey": "A"}, {"core": "coreB", "key": "B"}]},
"otherCoreFacetFilters": {},
"queries": {},
"rankQueries": {},
"resultsFrom": "coreA",
"sortKeys": [],
"unites": [],
},
"type": "ComposedQuery",
},
result.info,
)
def testAddRankQuery(self):
cq = ComposedQuery('coreA')
cq.setRankQuery('coreB', 'qB')
self.assertValidateRaisesValueError(cq, "No match set for cores ('coreA', 'coreB')")
cq.addMatch(dict(core='coreA', uniqueKey='kA'), dict(core='coreB', key='kB'))
self.assertEquals('qB', cq.rankQueryFor('coreB'))
cq.convertWith(coreB=lambda q: "converted_" + q)
self.assertEquals('converted_qB', cq.rankQueryFor('coreB'))
def testUniteMakesItTwoCoreQuery(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(3, result.total)
self.assertEquals(set(['A-QU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
def testMultipleJoinQueriesKeepsCachesWithinMaxSize(self):
for i in xrange(25):
self.addDocument(self.luceneB, identifier=str(i), keys=[('X', i)], fields=[('Y', str(i))])
for i in xrange(25):
q = ComposedQuery('coreA', query=MatchAllDocsQuery())
q.setCoreQuery(core='coreB', query=luceneQueryFromCql('Y=%s' % i))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'X'))
ignoredResult = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
def testAddDrilldownQuery(self):
cq = ComposedQuery('coreA')
cq.addDrilldownQuery('coreB', ('field', ['value']))
self.assertValidateRaisesValueError(cq, "No match set for cores ('coreA', 'coreB')")
cq.addMatch(dict(core='coreA', uniqueKey='kA'), dict(core='coreB', key='kB'))
self.assertEquals([('field', ['value'])], cq.drilldownQueriesFor('coreB'))
cq.convertWith(coreB=lambda q: "converted_" + q)
self.assertEquals([('field', ['value'])], cq.drilldownQueriesFor('coreB'))
def testAddRankQuery(self):
cq = ComposedQuery('coreA')
cq.setRankQuery('coreB', 'qB')
self.assertValidateRaisesValueError(cq, "No match set for cores ('coreA', 'coreB')")
cq.addMatch(dict(core='coreA', uniqueKey='kA'), dict(core='coreB', key='kB'))
self.assertEquals('qB', cq.rankQueryFor('coreB'))
cq.convertWith(coreB=lambda q: "converted_" + q)
self.assertEquals('converted_qB', cq.rankQueryFor('coreB'))
def testAddDrilldownQuery(self):
cq = ComposedQuery('coreA')
cq.addDrilldownQuery('coreB', ('field', ['value']))
self.assertValidateRaisesValueError(cq, "No match set for cores ('coreA', 'coreB')")
cq.addMatch(dict(core='coreA', uniqueKey='kA'), dict(core='coreB', key='kB'))
self.assertEquals([('field', ['value'])], cq.drilldownQueriesFor('coreB'))
cq.convertWith(coreB=lambda q: "converted_" + q)
self.assertEquals([('field', ['value'])], cq.drilldownQueriesFor('coreB'))
def testKeyName(self):
cq = ComposedQuery('coreA')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
cq.addFacet(core='coreB', facet='F0')
self.assertEquals('keyA', cq.keyName('coreA', 'coreB'))
self.assertEquals('keyB', cq.keyName('coreB', 'coreA'))
self.assertEquals(set(['keyA']), cq.keyNames('coreA'))
def testAddOtherCoreFacetFilter(self):
cq = ComposedQuery('coreA')
cq.addOtherCoreFacetFilter('coreB', 'field=value')
self.assertValidateRaisesValueError(cq, "No match set for cores ('coreA', 'coreB')")
cq.addMatch(dict(core='coreA', uniqueKey='kA'), dict(core='coreB', key='kB'))
self.assertEquals(['field=value'], cq.otherCoreFacetFiltersFor('coreB'))
cq.convertWith(coreB=lambda q: "converted_" + q)
self.assertEquals(['converted_field=value'], cq.otherCoreFacetFiltersFor('coreB'))
def testKeyFilterIgnoresKeysOutOfBoundsOfKeySet(self):
self.addDocument(self.luceneB, identifier=str(100), keys=[('B', 100)], fields=[]) # Force key to be much more than bits in long[] in FixedBitSet, so it must be OutOfBounds
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=MatchAllDocsQuery())
q.setCoreQuery(core='coreB', query=MatchAllDocsQuery())
q.addFacet(core='coreB', facet=dict(fieldname='cat_M', maxTerms=10))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(4, len(result.hits))
def testRankQuery(self):
q = ComposedQuery('coreA', query=MatchAllDocsQuery())
q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'))
q.setRankQuery(core='coreC', query=luceneQueryFromCql('S=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(4, result.total)
self.assertEquals([u'A-MQU', 'A-M', 'A-MU', u'A-MQ'], [hit.id for hit in result.hits])
def testUniteResultFromTwoIndexesCachedAfterUpdate(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[
dict(fieldname='cat_Q', maxTerms=10),
dict(fieldname='cat_U', maxTerms=10),
])
q.setCoreQuery(core='coreB', query=None, facets=[
dict(fieldname='cat_N', maxTerms=10),
dict(fieldname='cat_O', maxTerms=10),
])
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
resultOne = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(3, resultOne.total)
self.assertEquals([{
'terms': [
{'count': 3, 'term': u'true'}
], 'path': [], 'fieldname': u'cat_Q'
}, {
'terms': [
{'count': 2, 'term': u'true'},
{'count': 1, 'term': u'false'}
], 'path': [], 'fieldname': u'cat_U'
}, {
'terms': [
{'count': 2, 'term': u'true'}
], 'path': [], 'fieldname': u'cat_N'
}, {
'terms': [
{'count': 1, 'term': u'true'},
{'count': 1, 'term': u'false'},
], 'path': [], 'fieldname': u'cat_O'
}], resultOne.drilldownData)
self.addDocument(self.luceneA, identifier='A-MQU', keys=[('A', 8 )], fields=[('M', 'true' ), ('Q', 'false' ), ('U', 'true' ), ('S', '8')])
resultAgain = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(2, resultAgain.total)
self.assertEquals([{
'terms': [
{'count': 2, 'term': u'true'}
], 'path': [], 'fieldname': u'cat_Q'
}, {
'terms': [
{'count': 1, 'term': u'false'},
{'count': 1, 'term': u'true'},
], 'path': [], 'fieldname': u'cat_U'
}, {
'terms': [
{'count': 1, 'term': u'true'}
], 'path': [], 'fieldname': u'cat_N'
}, {
'terms': [
{'count': 1, 'term': u'true'}
], 'path': [], 'fieldname': u'cat_O'
}], resultAgain.drilldownData)
def testUnite(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query=None)
cq.setCoreQuery(core='coreB', query=None)
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery'))
self.assertEqual(1, len(cq.unites))
queries = list(cq.unites[0].queries())
self.assertEquals(({'query': 'AQuery', 'keyName': 'keyA', 'core': 'coreA'}, 'keyA'), queries[0])
self.assertEquals(({'query': 'anotherQuery', 'keyName': 'keyB', 'core': 'coreB'}, 'keyA'), queries[1])
def testValidateComposedQuery(self):
composedQuery = ComposedQuery('coreA')
composedQuery.setCoreQuery(core='coreA', query='Q0')
composedQuery.setCoreQuery(core='coreB', query='Q1')
self.assertValidateRaisesValueError(composedQuery, "No match set for cores ('coreA', 'coreB')")
composedQuery = ComposedQuery('coreA', query="A")
composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
composedQuery.validate()
self.assertEquals(1, composedQuery.numberOfUsedCores)
def testSameCoreInDifferentMatchesRequiredToHaveSameKeyForNow(self):
composedQuery = ComposedQuery('coreA', query='qA')
composedQuery.setCoreQuery('coreB', query='qB')
composedQuery.setCoreQuery('coreC', query='qC')
composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
self.assertRaises(
ValueError,
lambda: composedQuery.addMatch(dict(core='coreA', key='keyX'),
dict(core='coreC', key='keyC')))
def testValidateComposedQuery(self):
composedQuery = ComposedQuery('coreA')
composedQuery.setCoreQuery(core='coreA', query='Q0')
composedQuery.setCoreQuery(core='coreB', query='Q1')
self.assertValidateRaisesValueError(composedQuery, "No match set for cores ('coreA', 'coreB')")
composedQuery = ComposedQuery('coreA', query="A")
composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
composedQuery.validate()
self.assertEquals(1, composedQuery.numberOfUsedCores)
def testAtMostOneMultiCoreOr(self):
composedQuery = ComposedQuery('coreA')
composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreC', key='keyC'))
composedQuery.addUnite(dict(core='coreA', query='qA'),
dict(core='coreB', query='qB'))
self.assertRaises(
ValueError, lambda: composedQuery.addUnite(
dict(core='coreA', query='qA'), dict('coreC', query='qC')))
def testCachingCollectorsAfterDelete(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=MatchAllDocsQuery())
q.setCoreQuery(core='coreB', query=luceneQueryFromCql("N=true"))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
self.addDocument(self.luceneB, identifier='B-N>A-MQU', keys=[('B', 8 )], fields=[('N', 'true' ), ('O', 'false'), ('P', 'false')])
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(set([u'A-M', u'A-MU', u'A-MQ', u'A-MQU']), self.hitIds(result.hits))
consume(self.luceneB.delete(identifier='B-N>A-MU'))
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(set([u'A-M', u'A-MQ', u'A-MQU']), self.hitIds(result.hits))
def testConvertComposedQuery(self):
q = ComposedQuery('A')
q.setCoreQuery(core='A', query=cqlToExpression('valueAQ'))
q.setCoreQuery(core='B', query=cqlToExpression('valueBQ'))
q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B', key='keyB'))
q.addUnite(dict(core='A', query=cqlToExpression('fieldUA exact valueUA')), dict(core='B', query=cqlToExpression('fieldUB exact valueUB')))
q.validate()
consume(self.dna.any.executeComposedQuery(query=q))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
self.assertEquals("{'type': 'TermQuery', 'term': {'field': 'fieldA', 'value': u'valueaq'}, 'boost': 1.0}", repr(q.queryFor('A')))
self.assertEquals("{'type': 'TermQuery', 'term': {'field': 'fieldB', 'value': u'valuebq'}, 'boost': 1.0}", repr(q.queryFor('B')))
def testConvertComposedQuery(self):
q = ComposedQuery('A')
q.setCoreQuery(core='A', query=parseString('valueAQ'))
q.setCoreQuery(core='B', query=parseString('valueBQ'))
q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B', key='keyB'))
q.addUnite(dict(core='A', query=parseString('fieldUA exact valueUA')), dict(core='B', query=parseString('fieldUB exact valueUB')))
q.validate()
consume(self.dna.any.executeComposedQuery(query=q))
self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
self.assertEquals(repr(TermQuery(Term('fieldA', 'valueaq'))), repr(q.queryFor('A')))
self.assertEquals(repr(TermQuery(Term('fieldB', 'valuebq'))), repr(q.queryFor('B')))
def testAddOtherCoreFacetFilter(self):
cq = ComposedQuery('coreA')
cq.addOtherCoreFacetFilter('coreB', 'field=value')
self.assertValidateRaisesValueError(
cq, "No match set for cores ('coreA', 'coreB')")
cq.addMatch(dict(core='coreA', uniqueKey='kA'),
dict(core='coreB', key='kB'))
self.assertEquals(['field=value'],
cq.otherCoreFacetFiltersFor('coreB'))
cq.convertWith(coreB=lambda q: "converted_" + q)
self.assertEquals(['converted_field=value'],
cq.otherCoreFacetFiltersFor('coreB'))
def testJoinFacetFromBPointOfView(self):
q = ComposedQuery('coreB')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'))
q.setCoreQuery(core='coreB', query=None, facets=[
dict(fieldname='cat_N', maxTerms=10),
dict(fieldname='cat_O', maxTerms=10),
])
try:
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B'))
except ValueError, e:
self.assertEquals("Match for result core 'coreB' must have a uniqueKey specification.", str(e))
return
def testNullIteratorOfPForDeltaIsIgnoredInFinalKeySet(self):
q = ComposedQuery('coreA')
q.setCoreQuery(core='coreA', query=luceneQueryFromCql('N=no_match'))
q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'))
q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'UNKOWN'), dict(core='coreB', key=KEY_PREFIX+'UNKOWN'))
returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.luceneB.commit() # Force to write new segment; Old segment remains in seen list
self.addDocument(self.luceneB, identifier='new', keys=[], fields=[('ignored', 'true')]) # Add new document to force recreating finalKeySet
try:
result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
self.assertEquals(0, len(result.hits))
finally:
self.luceneB.delete(identifier='new')
def testConvertComposedQuery(self):
q = ComposedQuery('A')
q.setCoreQuery(core='A', query=parseString('valueAQ'))
q.setCoreQuery(core='B', query=parseString('valueBQ'))
q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B',
key='keyB'))
q.addUnite(dict(core='A', query=parseString('fieldUA exact valueUA')),
dict(core='B', query=parseString('fieldUB exact valueUB')))
q.validate()
consume(self.dna.any.executeComposedQuery(query=q))
self.assertEquals(['executeComposedQuery'],
self.observer.calledMethodNames())
self.assertEquals(repr(TermQuery(Term('fieldA', 'valueaq'))),
repr(q.queryFor('A')))
self.assertEquals(repr(TermQuery(Term('fieldB', 'valuebq'))),
repr(q.queryFor('B')))
def testMatchesMustAlwaysIncludeResultsFrom(self):
composedQuery = ComposedQuery('coreA', query='qA')
composedQuery.setCoreQuery('coreB', query='qB')
composedQuery.setCoreQuery('coreC', query='qC')
self.assertRaises(
ValueError,
lambda: composedQuery.addMatch(dict(core='coreB', key='keyB'),
dict(core='coreC', key='keyC')))
def testConvertAllQueries(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query='Q0')
cq.addFilterQuery('coreA', 'Q1')
cq.addFilterQuery('coreA', 'Q2')
cq.setCoreQuery(core='coreB', query='Q3')
cq.addFilterQuery('coreB', 'Q4')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
convertCoreA = lambda query: "Converted_A_{0}".format(query)
convertCoreB = lambda query: "Converted_B_{0}".format(query)
cq.convertWith(coreA=convertCoreA, coreB=convertCoreB)
self.assertEquals("Converted_A_Q0", cq.queryFor('coreA'))
self.assertEquals(["Converted_A_Q1", "Converted_A_Q2"], cq.filterQueriesFor('coreA'))
self.assertEquals("Converted_B_Q3", cq.queryFor('coreB'))
self.assertEquals(["Converted_B_Q4"], cq.filterQueriesFor('coreB'))
self.assertEquals([('coreA', 'Converted_A_Q5'), ('coreB', 'Converted_B_Q6')], cq.unites)
def testAsDictFromDict(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query='Q0')
cq.addFilterQuery(core='coreA', query='Q1')
cq.addFilterQuery(core='coreA', query='Q2')
cq.relationalFilter = '{"type": "madeUpJoinQuery"}'
cq.addFacet(core='coreA', facet='F0')
cq.addFacet(core='coreA', facet='F1')
cq.setCoreQuery(core='coreB', query='Q3')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='AQuery'),
dict(core='coreB', query='anotherQuery'))
cq.start = 0
cq.sortKeys = [dict(sortBy='field', sortDescending=True)]
cq.clustering = True
cq.clusteringConfig = {'clusteringEps': 0.2}
cq.rankQueryScoreRatio = 0.75
d = cq.asDict()
cq2 = ComposedQuery.fromDict(d)
self.assertEquals('coreA', cq2.resultsFrom)
self.assertEquals(0, cq2.start)
self.assertEquals(None, cq2.stop)
self.assertEquals(['Q0', 'Q1', 'Q2'], cq2.queriesFor('coreA'))
self.assertEquals('{"type": "madeUpJoinQuery"}', cq2.relationalFilter)
self.assertEquals(['F0', 'F1'], cq2.facetsFor('coreA'))
self.assertEquals('keyA', cq2.keyName('coreA', 'coreB'))
self.assertEquals('keyB', cq2.keyName('coreB', 'coreA'))
self.assertEqual(1, len(cq2.unites))
queries = list(cq2.unites[0].queries())
self.assertEquals(({
'core': 'coreA',
'keyName': 'keyA',
'query': 'AQuery'
}, 'keyA'), queries[0])
self.assertEquals(({
'core': 'coreB',
'keyName': 'keyB',
'query': 'anotherQuery'
}, 'keyA'), queries[1])
self.assertEquals({'clusteringEps': 0.2}, cq2.clusteringConfig)
self.assertEquals(0.75, cq2.rankQueryScoreRatio)
def testConvertComposedQuery(self):
q = ComposedQuery('A')
q.setCoreQuery(core='A', query=cqlToExpression('valueAQ'))
q.setCoreQuery(core='B', query=cqlToExpression('valueBQ'))
q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B',
key='keyB'))
q.addUnite(
dict(core='A', query=cqlToExpression('fieldUA exact valueUA')),
dict(core='B', query=cqlToExpression('fieldUB exact valueUB')))
q.validate()
consume(self.dna.any.executeComposedQuery(query=q))
self.assertEquals(['executeComposedQuery'],
self.observer.calledMethodNames())
self.assertEquals(
"{'type': 'TermQuery', 'term': {'field': 'fieldA', 'value': u'valueaq'}, 'boost': 1.0}",
repr(q.queryFor('A')))
self.assertEquals(
"{'type': 'TermQuery', 'term': {'field': 'fieldB', 'value': u'valuebq'}, 'boost': 1.0}",
repr(q.queryFor('B')))
def testUnite(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query=None)
cq.setCoreQuery(core='coreB', query=None)
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='AQuery'),
dict(core='coreB', query='anotherQuery'))
self.assertEqual(1, len(cq.unites))
queries = list(cq.unites[0].queries())
self.assertEquals(({
'query': 'AQuery',
'keyName': 'keyA',
'core': 'coreA'
}, 'keyA'), queries[0])
self.assertEquals(({
'query': 'anotherQuery',
'keyName': 'keyB',
'core': 'coreB'
}, 'keyA'), queries[1])
def testConversionOfComposedQuery(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA',
query=parseString('Q0'),
filterQueries=['Q1', 'Q2'],
facets=['F0', 'F1'])
cq.setCoreQuery(core='coreB', query='Q3', filterQueries=['Q4'])
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='AQuery'),
dict(core='coreB', query='anotherQuery'))
cq.start = 0
cq.sortKeys = [dict(sortBy='field', sortDescending=True)]
kwargs = {'q': cq}
dump = jsonDumpMessage(message='aMessage', **kwargs)
self.assertEquals(str, type(dump))
message, kwargs = jsonLoadMessage(dump)
self.assertEquals('aMessage', message)
cq2 = kwargs['q']
self.assertEquals(parseString('Q0'), cq2.queryFor('coreA'))
def testAsDictFromDict(self):
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query='Q0')
cq.addFilterQuery(core='coreA', query='Q1')
cq.addFilterQuery(core='coreA', query='Q2')
cq.addFacet(core='coreA', facet='F0')
cq.addFacet(core='coreA', facet='F1')
cq.setCoreQuery(core='coreB', query='Q3')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery'))
cq.start = 0
cq.sortKeys = [dict(sortBy='field', sortDescending=True)]
d = cq.asDict()
cq2 = ComposedQuery.fromDict(d)
self.assertEquals('coreA', cq2.resultsFrom)
self.assertEquals(0, cq2.start)
self.assertEquals(None, cq2.stop)
self.assertEquals(['Q0', 'Q1', 'Q2'], cq2.queriesFor('coreA'))
self.assertEquals(['F0', 'F1'], cq2.facetsFor('coreA'))
self.assertEquals('keyA', cq2.keyName('coreA'))
self.assertEquals('keyB', cq2.keyName('coreB'))
def testUniqueKeyDoesntMatchResultsFrom(self):
composedQuery = ComposedQuery('coreA', query='A').setCoreQuery('coreB', query='bQ')
self.assertRaises(ValueError, lambda: composedQuery.addMatch(dict(core='coreA', key='keyA'), dict(core='coreB', key='keyB')))
self.assertRaises(ValueError, lambda: composedQuery.addMatch(dict(core='coreA', key='keyA'), dict(core='coreB', uniqueKey='keyB')))
composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
composedQuery.validate()
composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', uniqueKey='keyB'))
composedQuery.validate()
def testOtherCoreTermQuery(self):
query = ComposedQuery('thisCore')
query.cores.add('otherCore')
query.addMatch(dict(core='thisCore', uniqueKey='A'),
dict(core='otherCore', uniqueKey='B'))
self.assertEquals(
{
"type":
"RelationalLuceneQuery", # should this not be 'joined' to own core somehow? (with MatchAllDocs)
"core": "otherCore",
"collectKeyName": "B",
"filterKeyName": "B",
"query": {
"type": "TermQuery",
"term": {
"field": "field",
"value": "value",
}
}
},
self._convert(QueryExpression.searchterm("otherCore.field", "=",
"value"),
composedQuery=query))
def testRepr(self):
class AQuery(object):
def __repr__(self):
return 'NOT USED'
def __str__(self):
return 'AQuery'
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query='Q0')
cq.addFilterQuery(core='coreA', query='Q1')
cq.addFilterQuery(core='coreA', query='Q2')
cq.relationalFilter = '{"type": "MadeUpJoinQuery"}'
cq.addFacet(core='coreA', facet='F0')
cq.addFacet(core='coreA', facet='F1')
cq.setCoreQuery(core='coreB', query='Q3')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query=AQuery()),
dict(core='coreB', query='anotherQuery'))
cq.start = 0
cq.sortKeys = [dict(sortBy='field', sortDescending=True)]
cq.storedFields = ['stored_field']
self.assertEquals(
{
'type': 'ComposedQuery',
'query': {
"cores": ["coreB", "coreA"],
"drilldownQueries": {},
"facets": {
"coreA": ["F0", "F1"]
},
"filterQueries": {
"coreA": ["Q1", "Q2"]
},
"matches": {
"coreA->coreB": [{
"core": "coreA",
"uniqueKey": "keyA"
}, {
"core": "coreB",
"key": "keyB"
}]
},
"otherCoreFacetFilters": {},
"queries": {
"coreA": "Q0",
"coreB": "Q3"
},
"rankQueries": {},
'relationalFilter':
'{"type": "MadeUpJoinQuery"}',
"resultsFrom":
"coreA",
"sortKeys": [{
"sortBy": "field",
"sortDescending": True
}],
"start":
0,
"storedFields": ['stored_field'],
"unites": [{
"A": ["coreA", "AQuery"],
"B": ["coreB", "anotherQuery"]
}]
}
}, cq.infoDict())
def testUniteQueriesFor(self):
cq = ComposedQuery('coreA')
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
self.assertEquals(['Q5'], cq.uniteQueriesFor('coreA'))
self.assertEquals(['Q6'], cq.uniteQueriesFor('coreB'))