def testComposedQuery(self): self.response = JsonDict({ "total": 887, "queryTime": 6, "hits": [{"id": "record:1", "score": 0.1234}] }).dumps() cq = ComposedQuery('coreA') q = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value")) cq.setCoreQuery("coreA", q) consume(self._multiLucene.executeComposedQuery(cq)) self.assertEqual(1, len(self.post)) self.assertEqual("/query/", self.post[0]['path']) self.assertEqual({ "_sortKeys": [], "resultsFrom": "coreA", "_matches": {}, "_facets": {}, "_otherCoreFacetFilters": {}, "_rankQueries": {}, "_drilldownQueries": {}, "_unites": [], "_queries": {"coreA": {"term": {"field": "field", "value": "value"}, "type": "TermQuery"}}, "cores": ["coreA"], "_filterQueries": {} }, loads(self.post[0]['data']))
def testIsSingleCoreQuery(self): cq = ComposedQuery('coreA') cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.setCoreQuery('coreA', query='A') self.assertTrue(cq.isSingleCoreQuery()) cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6')) self.assertFalse(cq.isSingleCoreQuery())
def testUnite(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=None) cq.setCoreQuery(core='coreB', query=None) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery')) self.assertEquals([('coreA', 'AQuery'), ('coreB', 'anotherQuery')], cq.unites)
def testKeyName(self): composedQuery = ComposedQuery('coreA') composedQuery.setCoreQuery(core='coreA', query=None) composedQuery.setCoreQuery(core='coreB', query=None) composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) self.assertEquals('keyA', composedQuery.keyName('coreA')) self.assertEquals('keyB', composedQuery.keyName('coreB'))
def testRemoteExecuteQuery(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[ Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5") ]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost cq = ComposedQuery('coreA') cq.setCoreQuery( core='coreA', query=parseString('query AND field=value'), filterQueries=[parseString('query=fiets')], facets=[{ 'fieldname': 'field', 'maxTerms': 5 }], ) cq.setCoreQuery(core='coreB', query=parseString('query=test')) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) result = returnValueFromGenerator( observable.any.executeComposedQuery(query=cq)) self.assertEquals(5, result.total) self.assertEquals([ Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5") ], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) message, kwargs = jsonLoadMessage(m.kwargs['body']) query = kwargs['query'] self.assertEquals('executeComposedQuery', message) self.assertEquals('coreA', query.resultsFrom) self.assertEquals([{ 'fieldname': 'field', 'maxTerms': 5 }], query.facetsFor('coreA'))
def testAddTypeAndMissingValueToSortField(self): self.response = JsonDict({"total": 887, "queryTime": 6, "hits": [{"id": "record:1", "score": 0.1234}]}).dumps() cq = ComposedQuery("coreA") q = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value")) cq.setCoreQuery("coreB", q) cq.sortKeys = [dict(sortBy="sortField", core="coreA", sortDescending=True)] cq.addMatch(dict(core="coreA", uniqueKey="A"), dict(core="coreB", key="B")) consume(self._multiLucene.executeComposedQuery(cq)) self.assertEqual( { "_sortKeys": [ { "core": "coreA", "sortBy": "sortField", "sortDescending": True, "type": "String", "missingValue": "STRING_FIRST", } ], "resultsFrom": "coreA", "_matches": {"coreA->coreB": [{"core": "coreA", "uniqueKey": "A"}, {"core": "coreB", "key": "B"}]}, "_facets": {}, "_otherCoreFacetFilters": {}, "_rankQueries": {}, "_drilldownQueries": {}, "_unites": [], "_queries": {"coreB": {"term": {"field": "field", "value": "value"}, "type": "TermQuery"}}, "cores": ["coreB", "coreA"], "_filterQueries": {}, }, loads(self.post[0]["data"]), )
def testUniteResultFromTwoIndexesCached(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[ dict(fieldname='cat_Q', maxTerms=10), dict(fieldname='cat_U', maxTerms=10), ]) q.setCoreQuery(core='coreB', query=None, facets=[ dict(fieldname='cat_N', maxTerms=10), dict(fieldname='cat_O', maxTerms=10), ]) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) resultOne = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('U=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=false')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) ignoredResult = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[ dict(fieldname='cat_Q', maxTerms=10), dict(fieldname='cat_U', maxTerms=10), ]) q.setCoreQuery(core='coreB', query=None, facets=[ dict(fieldname='cat_N', maxTerms=10), dict(fieldname='cat_O', maxTerms=10), ]) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) resultAgain = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(resultOne.total, resultAgain.total) self.assertEquals(resultOne.hits, resultAgain.hits) self.assertEquals(resultOne.drilldownData, resultAgain.drilldownData)
def testRemoteExecuteQuery(self): http = CallTrace('http') def httppost(*args, **kwargs): raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")]).asJson()) yield http.methods['httppost'] = httppost remote = LuceneRemote(host='host', port=1234, path='/path') observable = Observable() observable.addObserver(remote) remote._httppost = http.httppost cq = ComposedQuery('coreA') cq.setCoreQuery( core='coreA', query=parseString('query AND field=value'), filterQueries=[parseString('query=fiets')], facets=[{'fieldname': 'field', 'maxTerms':5}], ) cq.setCoreQuery(core='coreB', query=parseString('query=test')) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) result = returnValueFromGenerator(observable.any.executeComposedQuery(query=cq)) self.assertEquals(5, result.total) self.assertEquals([Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")], result.hits) self.assertEquals(['httppost'], http.calledMethodNames()) m = http.calledMethods[0] self.assertEquals('host', m.kwargs['host']) self.assertEquals(1234, m.kwargs['port']) self.assertEquals('/path/__lucene_remote__', m.kwargs['request']) self.assertEquals('application/json', m.kwargs['headers']['Content-Type']) message, kwargs = Conversion().jsonLoadMessage(m.kwargs['body']) query = kwargs['query'] self.assertEquals('executeComposedQuery', message) self.assertEquals('coreA', query.resultsFrom) self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], query.facetsFor('coreA'))
def testUniteMakesItTwoCoreQuery(self): cq = ComposedQuery('coreA') cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.setCoreQuery('coreA', query='A') cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6')) cq.validate() self.assertEquals(set(['coreA', 'coreB']), cq.cores)
def testConvertAllQueriesWithUnqualifiedTermFields(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query='Q0') cq.addFilterQuery('coreA', 'Q1') cq.addFilterQuery('coreA', 'Q2') cq.setCoreQuery(core='coreB', query='Q3') cq.addFilterQuery('coreB', 'Q4') cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6')) cq.unqualifiedTermFields = [('field0', 2.0), ('field1', 3.0)] convertCoreA = lambda query, unqualifiedTermFields=None, **kwargs: "Converted_A_{0}_{1}".format( query, not unqualifiedTermFields is None) convertCoreB = lambda query, **kwargs: "Converted_B_{0}".format(query) cq.convertWith(coreA=convertCoreA, coreB=convertCoreB) self.assertEquals("Converted_A_Q0_True", cq.queryFor('coreA')) self.assertEquals(["Converted_A_Q1_True", "Converted_A_Q2_True"], cq.filterQueriesFor('coreA')) self.assertEquals("Converted_B_Q3", cq.queryFor('coreB')) self.assertEquals(["Converted_B_Q4"], cq.filterQueriesFor('coreB')) self.assertEqual(1, len(cq.unites)) uniteQueries = list(cq.unites[0].queries()) self.assertEquals('Converted_A_Q5_True', uniteQueries[0][0]['query']) self.assertEquals('Converted_B_Q6', uniteQueries[1][0]['query'])
def testFilterQueries(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query='Q0') cq.addFilterQuery(core='coreA', query='Q1') cq.addFilterQuery(core='coreA', query='Q2') cq.addFacet(core='coreA', facet='F0') cq.addFacet(core='coreA', facet='F1') cq.setCoreQuery(core='coreB', query='Q3') cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery')) self.assertEquals(None, cq.stop) self.assertEquals(None, cq.start) self.assertEquals([], cq.sortKeys) cq.stop = 10 cq.start = 0 cq.sortKeys = [dict(sortBy='field', sortDescending=True)] self.assertEquals('Q0', cq.queryFor('coreA')) self.assertEquals(['Q1', 'Q2'], cq.filterQueriesFor('coreA')) self.assertEquals(['F0', 'F1'], cq.facetsFor('coreA')) self.assertEquals(10, cq.stop) self.assertEquals(0, cq.start) self.assertEquals([dict(sortBy='field', sortDescending=True)], cq.sortKeys)
def testJoinQuery(self): q = ComposedQuery('coreA', query=MatchAllDocsQuery()) q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(4, result.total) self.assertEquals(set(['A-M', 'A-MU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
def testAsDictFromDict(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query='Q0') cq.addFilterQuery(core='coreA', query='Q1') cq.addFilterQuery(core='coreA', query='Q2') cq.addFacet(core='coreA', facet='F0') cq.addFacet(core='coreA', facet='F1') cq.setCoreQuery(core='coreB', query='Q3') cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery')) cq.start = 0 cq.sortKeys = [dict(sortBy='field', sortDescending=True)] cq.clustering = True cq.clusteringConfig = {'clusteringEps': 0.2} d = cq.asDict() cq2 = ComposedQuery.fromDict(d) self.assertEquals('coreA', cq2.resultsFrom) self.assertEquals(0, cq2.start) self.assertEquals(None, cq2.stop) self.assertEquals(['Q0', 'Q1', 'Q2'], cq2.queriesFor('coreA')) self.assertEquals(['F0', 'F1'], cq2.facetsFor('coreA')) self.assertEquals('keyA', cq2.keyName('coreA', 'coreB')) self.assertEquals('keyB', cq2.keyName('coreB', 'coreA')) self.assertEqual(1, len(cq2.unites)) queries = list(cq2.unites[0].queries()) self.assertEquals(({'core': 'coreA', 'keyName': 'keyA', 'query': 'AQuery'}, 'keyA'), queries[0]) self.assertEquals(({'core': 'coreB', 'keyName': 'keyB', 'query': 'anotherQuery'}, 'keyA'), queries[1]) self.assertEquals({'clusteringEps': 0.2}, cq2.clusteringConfig)
def testJoinFacetAndQuery(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=None) q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'), facets=[ dict(fieldname='cat_N', maxTerms=10), dict(fieldname='cat_O', maxTerms=10), ]) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(query=q)) self.assertEquals(4, result.total) self.assertEquals(set(['A-M', 'A-MU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits)) self.assertEquals([{ 'terms': [ {'count': 4, 'term': u'true'}, ], 'path': [], 'fieldname': u'cat_N' }, { 'terms': [ {'count': 2, 'term': u'true'}, {'count': 2, 'term': u'false'}, ], 'path': [], 'fieldname': u'cat_O' }], result.drilldownData)
def testAddTypeAndMissingValueToSortField(self): self.response = JsonDict({ "total": 887, "queryTime": 6, "hits": [{"id": "record:1", "score": 0.1234}] }).dumps() cq = ComposedQuery('coreA') q = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value")) cq.setCoreQuery('coreB', q) cq.sortKeys = [dict(sortBy='sortField', core='coreA', sortDescending=True)] cq.addMatch(dict(core='coreA', uniqueKey='A'), dict(core='coreB', key='B')) consume(self._multiLucene.executeComposedQuery(cq)) self.assertEqual({ "_sortKeys": [{'core': 'coreA', 'sortBy': 'sortField', 'sortDescending': True, 'type': 'String', 'missingValue': 'STRING_FIRST'}], "resultsFrom": "coreA", '_matches': {'coreA->coreB': [{'core': 'coreA', 'uniqueKey': 'A'}, {'core': 'coreB', 'key': 'B'}]}, "_facets": {}, "_otherCoreFacetFilters": {}, "_rankQueries": {}, "_drilldownQueries": {}, "_unites": [], '_queries': {'coreB': {'term': {'field': 'field', 'value': 'value'}, 'type': 'TermQuery'}}, "cores": ["coreB", "coreA"], "_filterQueries": {} }, loads(self.post[0]['data']))
def testJoinQueryOnOptionalKeyOtherSide(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'D')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(1, result.total) self.assertEquals(set(['A-M']), self.hitIds(result.hits))
def testMultipleJoinQueriesKeepsCachesWithinMaxSize(self): for i in xrange(25): self.addDocument(self.luceneB, identifier=str(i), keys=[('X', i)], fields=[('Y', str(i))]) for i in xrange(25): q = ComposedQuery('coreA', query=MatchAllDocsQuery()) q.setCoreQuery(core='coreB', query=luceneQueryFromCql('Y=%s' % i)) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'X')) ignoredResult = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
def testMatchesMustAlwaysIncludeResultsFrom(self): composedQuery = ComposedQuery('coreA', query='qA') composedQuery.setCoreQuery('coreB', query='qB') composedQuery.setCoreQuery('coreC', query='qC') self.assertRaises( ValueError, lambda: composedQuery.addMatch(dict(core='coreB', key='keyB'), dict(core='coreC', key='keyC')))
def testUniteMakesItTwoCoreQuery(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(3, result.total) self.assertEquals(set(['A-QU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
def testKeyFilterIgnoresKeysOutOfBoundsOfKeySet(self): self.addDocument(self.luceneB, identifier=str(100), keys=[('B', 100)], fields=[]) # Force key to be much more than bits in long[] in FixedBitSet, so it must be OutOfBounds q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=MatchAllDocsQuery()) q.setCoreQuery(core='coreB', query=MatchAllDocsQuery()) q.addFacet(core='coreB', facet=dict(fieldname='cat_M', maxTerms=10)) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(4, len(result.hits))
def testRankQuery(self): q = ComposedQuery('coreA', query=MatchAllDocsQuery()) q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true')) q.setRankQuery(core='coreC', query=luceneQueryFromCql('S=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(4, result.total) self.assertEquals([u'A-MQU', 'A-M', 'A-MU', u'A-MQ'], [hit.id for hit in result.hits])
def testQueryOneIndexWithComposedQuery(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(cq)) self.assertEquals(set(['A-Q', 'A-QU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits)) cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), filterQueries=[luceneQueryFromCql('M=true')]) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(cq)) self.assertEquals(set(['A-MQ', 'A-MQU']), self.hitIds(result.hits))
def testUniteResultFromTwoIndexesCachedAfterUpdate(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[ dict(fieldname='cat_Q', maxTerms=10), dict(fieldname='cat_U', maxTerms=10), ]) q.setCoreQuery(core='coreB', query=None, facets=[ dict(fieldname='cat_N', maxTerms=10), dict(fieldname='cat_O', maxTerms=10), ]) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true'))) resultOne = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(3, resultOne.total) self.assertEquals([{ 'terms': [ {'count': 3, 'term': u'true'} ], 'path': [], 'fieldname': u'cat_Q' }, { 'terms': [ {'count': 2, 'term': u'true'}, {'count': 1, 'term': u'false'} ], 'path': [], 'fieldname': u'cat_U' }, { 'terms': [ {'count': 2, 'term': u'true'} ], 'path': [], 'fieldname': u'cat_N' }, { 'terms': [ {'count': 1, 'term': u'true'}, {'count': 1, 'term': u'false'}, ], 'path': [], 'fieldname': u'cat_O' }], resultOne.drilldownData) self.addDocument(self.luceneA, identifier='A-MQU', keys=[('A', 8 )], fields=[('M', 'true' ), ('Q', 'false' ), ('U', 'true' ), ('S', '8')]) resultAgain = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(2, resultAgain.total) self.assertEquals([{ 'terms': [ {'count': 2, 'term': u'true'} ], 'path': [], 'fieldname': u'cat_Q' }, { 'terms': [ {'count': 1, 'term': u'false'}, {'count': 1, 'term': u'true'}, ], 'path': [], 'fieldname': u'cat_U' }, { 'terms': [ {'count': 1, 'term': u'true'} ], 'path': [], 'fieldname': u'cat_N' }, { 'terms': [ {'count': 1, 'term': u'true'} ], 'path': [], 'fieldname': u'cat_O' }], resultAgain.drilldownData)
def testValidateComposedQuery(self): composedQuery = ComposedQuery('coreA') composedQuery.setCoreQuery(core='coreA', query='Q0') composedQuery.setCoreQuery(core='coreB', query='Q1') self.assertValidateRaisesValueError(composedQuery, "No match set for cores ('coreA', 'coreB')") composedQuery = ComposedQuery('coreA', query="A") composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) composedQuery.validate() self.assertEquals(1, composedQuery.numberOfUsedCores)
def testAddFilterQueryAfterConversion(self): cq = ComposedQuery('coreA') cq.setCoreQuery('coreA', query='A') cq.convertWith(coreA=lambda q, **kwargs: "converted_" + q) self.assertEquals('converted_A', cq.queryFor('coreA')) # Assert the following does not raise KeyError cq.addFilterQuery('coreA', 'field=value') cq.addFacet('coreA', 'F0') cq.addDrilldownQuery('coreA', 'drilldownQuery') cq.addOtherCoreFacetFilter('coreA', 'q')
def testSameCoreInDifferentMatchesRequiredToHaveSameKeyForNow(self): composedQuery = ComposedQuery('coreA', query='qA') composedQuery.setCoreQuery('coreB', query='qB') composedQuery.setCoreQuery('coreC', query='qC') composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) self.assertRaises( ValueError, lambda: composedQuery.addMatch(dict(core='coreA', key='keyX'), dict(core='coreC', key='keyC')))
def testUnite(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=None) cq.setCoreQuery(core='coreB', query=None) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery')) self.assertEqual(1, len(cq.unites)) queries = list(cq.unites[0].queries()) self.assertEquals(({'query': 'AQuery', 'keyName': 'keyA', 'core': 'coreA'}, 'keyA'), queries[0]) self.assertEquals(({'query': 'anotherQuery', 'keyName': 'keyB', 'core': 'coreB'}, 'keyA'), queries[1])
def testAddFilterQueryAfterConversion(self): cq = ComposedQuery('coreA') cq.setCoreQuery('coreA', query='A') cq.convertWith(coreA=lambda q: "converted_" + q) self.assertEquals('converted_A', cq.queryFor('coreA')) # Assert the following does not raise KeyError cq.addFilterQuery('coreA', 'field=value') cq.addFacet('coreA', 'F0') cq.addDrilldownQuery('coreA', 'drilldownQuery') cq.addOtherCoreFacetFilter('coreA', 'q')
def testConvertComposedQuery(self): q = ComposedQuery('A') q.setCoreQuery(core='A', query=cqlToExpression('valueAQ')) q.setCoreQuery(core='B', query=cqlToExpression('valueBQ')) q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B', key='keyB')) q.addUnite(dict(core='A', query=cqlToExpression('fieldUA exact valueUA')), dict(core='B', query=cqlToExpression('fieldUB exact valueUB'))) q.validate() consume(self.dna.any.executeComposedQuery(query=q)) self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames()) self.assertEquals("{'type': 'TermQuery', 'term': {'field': 'fieldA', 'value': u'valueaq'}, 'boost': 1.0}", repr(q.queryFor('A'))) self.assertEquals("{'type': 'TermQuery', 'term': {'field': 'fieldB', 'value': u'valuebq'}, 'boost': 1.0}", repr(q.queryFor('B')))
def testCachingCollectorsAfterDelete(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=MatchAllDocsQuery()) q.setCoreQuery(core='coreB', query=luceneQueryFromCql("N=true")) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) self.addDocument(self.luceneB, identifier='B-N>A-MQU', keys=[('B', 8 )], fields=[('N', 'true' ), ('O', 'false'), ('P', 'false')]) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(set([u'A-M', u'A-MU', u'A-MQ', u'A-MQU']), self.hitIds(result.hits)) consume(self.luceneB.delete(identifier='B-N>A-MU')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(set([u'A-M', u'A-MQ', u'A-MQU']), self.hitIds(result.hits))
def testConvertComposedQuery(self): q = ComposedQuery('A') q.setCoreQuery(core='A', query=parseString('valueAQ')) q.setCoreQuery(core='B', query=parseString('valueBQ')) q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B', key='keyB')) q.addUnite(dict(core='A', query=parseString('fieldUA exact valueUA')), dict(core='B', query=parseString('fieldUB exact valueUB'))) q.validate() consume(self.dna.any.executeComposedQuery(query=q)) self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames()) self.assertEquals(repr(TermQuery(Term('fieldA', 'valueaq'))), repr(q.queryFor('A'))) self.assertEquals(repr(TermQuery(Term('fieldB', 'valuebq'))), repr(q.queryFor('B')))
def testJoinFacetFromBPointOfView(self): q = ComposedQuery('coreB') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true')) q.setCoreQuery(core='coreB', query=None, facets=[ dict(fieldname='cat_N', maxTerms=10), dict(fieldname='cat_O', maxTerms=10), ]) try: q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B')) except ValueError, e: self.assertEquals("Match for result core 'coreB' must have a uniqueKey specification.", str(e)) return
def testNullIteratorOfPForDeltaIsIgnoredInFinalKeySet(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreA', query=luceneQueryFromCql('N=no_match')) q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'UNKOWN'), dict(core='coreB', key=KEY_PREFIX+'UNKOWN')) returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.luceneB.commit() # Force to write new segment; Old segment remains in seen list self.addDocument(self.luceneB, identifier='new', keys=[], fields=[('ignored', 'true')]) # Add new document to force recreating finalKeySet try: result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(0, len(result.hits)) finally: self.luceneB.delete(identifier='new')
def testJoinSort(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=MatchAllDocsQuery()) cq.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) cq.addSortKey({'sortBy': 'T', 'sortDescending': False, 'core': 'coreB'}) cq.addSortKey({'sortBy': 'S', 'sortDescending': False, 'core': 'coreA'}) result = retval(self.dna.any.executeComposedQuery(cq)) self.assertEqual(['A-M', 'A-MU', 'A-MQ', 'A-MQU', 'A', 'A-U', 'A-Q', 'A-QU'], [hit.id for hit in result.hits]) cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=MatchAllDocsQuery()) cq.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) cq.addSortKey({'sortBy': 'T', 'sortDescending': True, 'core': 'coreB'}) cq.addSortKey({'sortBy': 'S', 'sortDescending': False, 'core': 'coreA'}) result = retval(self.dna.any.executeComposedQuery(cq)) self.assertEqual(['A-MQU', 'A-MQ', 'A-MU', 'A-M', 'A', 'A-U', 'A-Q', 'A-QU'], [hit.id for hit in result.hits]) cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=MatchAllDocsQuery()) cq.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) cq.addSortKey({'sortBy': 'intField', 'sortDescending': True, 'core': 'coreB'}) cq.addSortKey({'sortBy': 'S', 'sortDescending': False, 'core': 'coreA'}) result = retval(self.dna.any.executeComposedQuery(cq)) self.assertEqual(['A-MQU', 'A-MQ', 'A-MU', 'A-M', 'A', 'A-U', 'A-Q', 'A-QU'], [hit.id for hit in result.hits]) cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=MatchAllDocsQuery()) cq.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) cq.addSortKey({'sortBy': 'intField', 'sortDescending': True, 'core': 'coreB', 'missingValue': 20}) cq.addSortKey({'sortBy': 'S', 'sortDescending': False, 'core': 'coreA'}) result = retval(self.dna.any.executeComposedQuery(cq)) self.assertEqual(['A', 'A-U', 'A-Q', 'A-QU', 'A-MQU', 'A-MQ', 'A-MU', 'A-M'], [hit.id for hit in result.hits])
def testJoinQueryThreeCores(self): q = ComposedQuery('coreA') q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true')) q.setCoreQuery(core='coreC', query=luceneQueryFromCql('R=true')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B')) q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C')) q.addFacet(core='coreA', facet=dict(fieldname='cat_M', maxTerms=10)) q.addFacet(core='coreB', facet=dict(fieldname='cat_N', maxTerms=10)) q.addFacet(core='coreC', facet=dict(fieldname='cat_R', maxTerms=10)) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q)) self.assertEquals(1, result.total) self.assertEquals(set(['A-M']), self.hitIds(result.hits)) self.assertEquals([ {'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_M'}, {'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_N'}, {'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_R'}, ], result.drilldownData)
def testJoinFacetWillNotFilter(self): query = ComposedQuery('coreA') query.setCoreQuery(core='coreA', query=None) query.setCoreQuery(core='coreB', query=None, facets=[ dict(fieldname='cat_N', maxTerms=10), ]) query.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B')) result = returnValueFromGenerator(self.dna.any.executeComposedQuery(query=query)) self.assertEquals(8, result.total) self.assertEquals([{ 'terms': [ {'count': 4, 'term': u'true'}, {'count': 4, 'term': u'false'}, ], 'path': [], 'fieldname': u'cat_N' }], result.drilldownData)
def testConvertAllQueries(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query='Q0') cq.addFilterQuery('coreA', 'Q1') cq.addFilterQuery('coreA', 'Q2') cq.setCoreQuery(core='coreB', query='Q3') cq.addFilterQuery('coreB', 'Q4') cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6')) convertCoreA = lambda query: "Converted_A_{0}".format(query) convertCoreB = lambda query: "Converted_B_{0}".format(query) cq.convertWith(coreA=convertCoreA, coreB=convertCoreB) self.assertEquals("Converted_A_Q0", cq.queryFor('coreA')) self.assertEquals(["Converted_A_Q1", "Converted_A_Q2"], cq.filterQueriesFor('coreA')) self.assertEquals("Converted_B_Q3", cq.queryFor('coreB')) self.assertEquals(["Converted_B_Q4"], cq.filterQueriesFor('coreB')) self.assertEquals([('coreA', 'Converted_A_Q5'), ('coreB', 'Converted_B_Q6')], cq.unites)
def testAsDictFromDict(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query='Q0') cq.addFilterQuery(core='coreA', query='Q1') cq.addFilterQuery(core='coreA', query='Q2') cq.relationalFilter = '{"type": "madeUpJoinQuery"}' cq.addFacet(core='coreA', facet='F0') cq.addFacet(core='coreA', facet='F1') cq.setCoreQuery(core='coreB', query='Q3') cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery')) cq.start = 0 cq.sortKeys = [dict(sortBy='field', sortDescending=True)] cq.clustering = True cq.clusteringConfig = {'clusteringEps': 0.2} cq.rankQueryScoreRatio = 0.75 d = cq.asDict() cq2 = ComposedQuery.fromDict(d) self.assertEquals('coreA', cq2.resultsFrom) self.assertEquals(0, cq2.start) self.assertEquals(None, cq2.stop) self.assertEquals(['Q0', 'Q1', 'Q2'], cq2.queriesFor('coreA')) self.assertEquals('{"type": "madeUpJoinQuery"}', cq2.relationalFilter) self.assertEquals(['F0', 'F1'], cq2.facetsFor('coreA')) self.assertEquals('keyA', cq2.keyName('coreA', 'coreB')) self.assertEquals('keyB', cq2.keyName('coreB', 'coreA')) self.assertEqual(1, len(cq2.unites)) queries = list(cq2.unites[0].queries()) self.assertEquals(({ 'core': 'coreA', 'keyName': 'keyA', 'query': 'AQuery' }, 'keyA'), queries[0]) self.assertEquals(({ 'core': 'coreB', 'keyName': 'keyB', 'query': 'anotherQuery' }, 'keyA'), queries[1]) self.assertEquals({'clusteringEps': 0.2}, cq2.clusteringConfig) self.assertEquals(0.75, cq2.rankQueryScoreRatio)
def testConvertComposedQuery(self): q = ComposedQuery('A') q.setCoreQuery(core='A', query=cqlToExpression('valueAQ')) q.setCoreQuery(core='B', query=cqlToExpression('valueBQ')) q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B', key='keyB')) q.addUnite( dict(core='A', query=cqlToExpression('fieldUA exact valueUA')), dict(core='B', query=cqlToExpression('fieldUB exact valueUB'))) q.validate() consume(self.dna.any.executeComposedQuery(query=q)) self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames()) self.assertEquals( "{'type': 'TermQuery', 'term': {'field': 'fieldA', 'value': u'valueaq'}, 'boost': 1.0}", repr(q.queryFor('A'))) self.assertEquals( "{'type': 'TermQuery', 'term': {'field': 'fieldB', 'value': u'valuebq'}, 'boost': 1.0}", repr(q.queryFor('B')))
def testUnite(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=None) cq.setCoreQuery(core='coreB', query=None) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery')) self.assertEqual(1, len(cq.unites)) queries = list(cq.unites[0].queries()) self.assertEquals(({ 'query': 'AQuery', 'keyName': 'keyA', 'core': 'coreA' }, 'keyA'), queries[0]) self.assertEquals(({ 'query': 'anotherQuery', 'keyName': 'keyB', 'core': 'coreB' }, 'keyA'), queries[1])
def testConversionOfComposedQuery(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=parseString('Q0'), filterQueries=['Q1', 'Q2'], facets=['F0', 'F1']) cq.setCoreQuery(core='coreB', query='Q3', filterQueries=['Q4']) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery')) cq.start = 0 cq.sortKeys = [dict(sortBy='field', sortDescending=True)] kwargs = {'q': cq} dump = jsonDumpMessage(message='aMessage', **kwargs) self.assertEquals(str, type(dump)) message, kwargs = jsonLoadMessage(dump) self.assertEquals('aMessage', message) cq2 = kwargs['q'] self.assertEquals(parseString('Q0'), cq2.queryFor('coreA'))
def testAsDictFromDict(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query='Q0') cq.addFilterQuery(core='coreA', query='Q1') cq.addFilterQuery(core='coreA', query='Q2') cq.addFacet(core='coreA', facet='F0') cq.addFacet(core='coreA', facet='F1') cq.setCoreQuery(core='coreB', query='Q3') cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery')) cq.start = 0 cq.sortKeys = [dict(sortBy='field', sortDescending=True)] d = cq.asDict() cq2 = ComposedQuery.fromDict(d) self.assertEquals('coreA', cq2.resultsFrom) self.assertEquals(0, cq2.start) self.assertEquals(None, cq2.stop) self.assertEquals(['Q0', 'Q1', 'Q2'], cq2.queriesFor('coreA')) self.assertEquals(['F0', 'F1'], cq2.facetsFor('coreA')) self.assertEquals('keyA', cq2.keyName('coreA')) self.assertEquals('keyB', cq2.keyName('coreB'))
def testRepr(self): class AQuery(object): def __repr__(self): return 'NOT USED' def __str__(self): return 'AQuery' cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query='Q0') cq.addFilterQuery(core='coreA', query='Q1') cq.addFilterQuery(core='coreA', query='Q2') cq.relationalFilter = '{"type": "MadeUpJoinQuery"}' cq.addFacet(core='coreA', facet='F0') cq.addFacet(core='coreA', facet='F1') cq.setCoreQuery(core='coreB', query='Q3') cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query=AQuery()), dict(core='coreB', query='anotherQuery')) cq.start = 0 cq.sortKeys = [dict(sortBy='field', sortDescending=True)] cq.storedFields = ['stored_field'] self.assertEquals( { 'type': 'ComposedQuery', 'query': { "cores": ["coreB", "coreA"], "drilldownQueries": {}, "facets": { "coreA": ["F0", "F1"] }, "filterQueries": { "coreA": ["Q1", "Q2"] }, "matches": { "coreA->coreB": [{ "core": "coreA", "uniqueKey": "keyA" }, { "core": "coreB", "key": "keyB" }] }, "otherCoreFacetFilters": {}, "queries": { "coreA": "Q0", "coreB": "Q3" }, "rankQueries": {}, 'relationalFilter': '{"type": "MadeUpJoinQuery"}', "resultsFrom": "coreA", "sortKeys": [{ "sortBy": "field", "sortDescending": True }], "start": 0, "storedFields": ['stored_field'], "unites": [{ "A": ["coreA", "AQuery"], "B": ["coreB", "anotherQuery"] }] } }, cq.infoDict())
def testSingleCoreQuery(self): cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query='Q0') cq.validate() self.assertEquals(1, cq.numberOfUsedCores)