Пример #1
0
 def testJoinQueryWithFilters(self):
     q = ComposedQuery('coreA')
     q.addFilterQuery('coreB', query=luceneQueryFromCql('N=true'))
     q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
     result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
     self.assertEquals(4, result.total)
     self.assertEquals(set(['A-M', 'A-MU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
Пример #2
0
    def testInfoOnQuery(self):
        self.response = JsonDict({"total": 887, "queryTime": 6, "hits": [{"id": "record:1", "score": 0.1234}]}).dumps()

        q = ComposedQuery("coreA")
        q.addFilterQuery("coreB", query="N=true")
        q.addMatch(dict(core="coreA", uniqueKey="A"), dict(core="coreB", key="B"))
        result = retval(self._multiLucene.executeComposedQuery(q))
        self.assertEquals(
            {
                "query": {
                    "cores": ["coreB", "coreA"],
                    "drilldownQueries": {},
                    "facets": {},
                    "filterQueries": {"coreB": ["N=true"]},
                    "matches": {"coreA->coreB": [{"core": "coreA", "uniqueKey": "A"}, {"core": "coreB", "key": "B"}]},
                    "otherCoreFacetFilters": {},
                    "queries": {},
                    "rankQueries": {},
                    "resultsFrom": "coreA",
                    "sortKeys": [],
                    "unites": [],
                },
                "type": "ComposedQuery",
            },
            result.info,
        )
Пример #3
0
 def XXX_NOT_YET_IMPLEMENTED_testRankQueryInSingleCoreQuery(self):
     q = ComposedQuery('coreA', query=MatchAllDocsQuery())
     q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
     q.setRankQuery(core='coreA', query=luceneQueryFromCql('Q=true'))
     result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
     self.assertEquals(8, result.total)
     self.assertEquals([u'A-Q', u'A-QU', u'A-MQ', u'A-MQU', u'A', u'A-U', u'A-M', u'A-MU'], [hit.id for hit in result.hits])
Пример #4
0
 def testJoinQueryOnOptionalKeyOtherSide(self):
     q = ComposedQuery('coreA')
     q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'))
     q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'D'))
     result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
     self.assertEquals(1, result.total)
     self.assertEquals(set(['A-M']), self.hitIds(result.hits))
Пример #5
0
 def testIsSingleCoreQuery(self):
     cq = ComposedQuery('coreA')
     cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
     cq.setCoreQuery('coreA', query='A')
     self.assertTrue(cq.isSingleCoreQuery())
     cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
     self.assertFalse(cq.isSingleCoreQuery())
Пример #6
0
 def testUniteMakesItTwoCoreQuery(self):
     cq = ComposedQuery('coreA')
     cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
     cq.setCoreQuery('coreA', query='A')
     cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
     cq.validate()
     self.assertEquals(set(['coreA', 'coreB']), cq.cores)
Пример #7
0
    def testComposedQuery(self):
        self.response = JsonDict({
                "total": 887,
                "queryTime": 6,
                "hits": [{"id": "record:1", "score": 0.1234}]
            }).dumps()

        cq = ComposedQuery('coreA')
        q = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
        cq.setCoreQuery("coreA", q)

        consume(self._multiLucene.executeComposedQuery(cq))
        self.assertEqual(1, len(self.post))
        self.assertEqual("/query/", self.post[0]['path'])
        self.assertEqual({
                "_sortKeys": [],
                "resultsFrom": "coreA",
                "_matches": {},
                "_facets": {},
                "_otherCoreFacetFilters": {},
                "_rankQueries": {},
                "_drilldownQueries": {},
                "_unites": [],
                "_queries": {"coreA": {"term": {"field": "field", "value": "value"}, "type": "TermQuery"}},
                "cores": ["coreA"],
                "_filterQueries": {}
            }, loads(self.post[0]['data']))
 def testMatchesMustAlwaysIncludeResultsFrom(self):
     composedQuery = ComposedQuery('coreA', query='qA')
     composedQuery.setCoreQuery('coreB', query='qB')
     composedQuery.setCoreQuery('coreC', query='qC')
     self.assertRaises(
         ValueError,
         lambda: composedQuery.addMatch(dict(core='coreB', key='keyB'),
                                        dict(core='coreC', key='keyC')))
Пример #9
0
 def testAddDrilldownQuery(self):
     cq = ComposedQuery('coreA')
     cq.addDrilldownQuery('coreB', ('field', ['value']))
     self.assertValidateRaisesValueError(cq, "No match set for cores ('coreA', 'coreB')")
     cq.addMatch(dict(core='coreA', uniqueKey='kA'), dict(core='coreB', key='kB'))
     self.assertEquals([('field', ['value'])], cq.drilldownQueriesFor('coreB'))
     cq.convertWith(coreB=lambda q: "converted_" + q)
     self.assertEquals([('field', ['value'])], cq.drilldownQueriesFor('coreB'))
Пример #10
0
 def testAddRankQuery(self):
     cq = ComposedQuery('coreA')
     cq.setRankQuery('coreB', 'qB')
     self.assertValidateRaisesValueError(cq, "No match set for cores ('coreA', 'coreB')")
     cq.addMatch(dict(core='coreA', uniqueKey='kA'), dict(core='coreB', key='kB'))
     self.assertEquals('qB', cq.rankQueryFor('coreB'))
     cq.convertWith(coreB=lambda q: "converted_" + q)
     self.assertEquals('converted_qB', cq.rankQueryFor('coreB'))
Пример #11
0
 def testMultipleJoinQueriesKeepsCachesWithinMaxSize(self):
     for i in xrange(25):
         self.addDocument(self.luceneB, identifier=str(i), keys=[('X', i)], fields=[('Y', str(i))])
     for i in xrange(25):
         q = ComposedQuery('coreA', query=MatchAllDocsQuery())
         q.setCoreQuery(core='coreB', query=luceneQueryFromCql('Y=%s' % i))
         q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'X'))
         ignoredResult = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
Пример #12
0
    def testAddFacetIncremental(self):
        cq = ComposedQuery('coreA')
        cq.addFacet(core='coreA', facet=dict(fieldname='Q1', maxTerms=10))
        cq.addFacet(core='coreA', facet=dict(fieldname='Q2', maxTerms=10))

        self.assertEquals([
            dict(fieldname='Q1', maxTerms=10),
            dict(fieldname='Q2', maxTerms=10)
        ], cq.facetsFor('coreA'))
 def testKeyNamesDifferPerCore(self):
     cq = ComposedQuery('coreA')
     cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
     cq.addMatch(dict(core='coreA', uniqueKey='keyAC'), dict(core='coreC', key='keyC'))
     cq.addFacet(core='coreB', facet='F0')
     cq.addFacet(core='coreC', facet='F1')
     self.assertEquals('keyAC', cq.keyName('coreA', 'coreC'))
     self.assertEquals('keyC', cq.keyName('coreC', 'coreA'))
     self.assertEquals(set(['keyA', 'keyAC']), cq.keyNames('coreA'))
Пример #14
0
    def testValidateComposedQuery(self):
        composedQuery = ComposedQuery('coreA')
        composedQuery.setCoreQuery(core='coreA', query='Q0')
        composedQuery.setCoreQuery(core='coreB', query='Q1')
        self.assertValidateRaisesValueError(composedQuery, "No match set for cores ('coreA', 'coreB')")

        composedQuery = ComposedQuery('coreA', query="A")
        composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
        composedQuery.validate()
        self.assertEquals(1, composedQuery.numberOfUsedCores)
 def testIsSingleCoreQuery(self):
     cq = ComposedQuery('coreA')
     cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
     cq.setCoreQuery('coreA', query='A')
     self.assertTrue(cq.isSingleCoreQuery())
     cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
     self.assertFalse(cq.isSingleCoreQuery())
 def testUniteMakesItTwoCoreQuery(self):
     cq = ComposedQuery('coreA')
     cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
     cq.setCoreQuery('coreA', query='A')
     cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
     cq.validate()
     self.assertEquals(set(['coreA', 'coreB']), cq.cores)
 def testConvertComposedQuery(self):
     q = ComposedQuery('A')
     q.setCoreQuery(core='A', query=parseString('valueAQ'))
     q.setCoreQuery(core='B', query=parseString('valueBQ'))
     q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B', key='keyB'))
     q.addUnite(dict(core='A', query=parseString('fieldUA exact valueUA')), dict(core='B', query=parseString('fieldUB exact valueUB')))
     q.validate()
     consume(self.dna.any.executeComposedQuery(query=q))
     self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
     self.assertEquals(repr(TermQuery(Term('fieldA', 'valueaq'))), repr(q.queryFor('A')))
     self.assertEquals(repr(TermQuery(Term('fieldB', 'valuebq'))), repr(q.queryFor('B')))
 def testAddOtherCoreFacetFilter(self):
     cq = ComposedQuery('coreA')
     cq.addOtherCoreFacetFilter('coreB', 'field=value')
     self.assertValidateRaisesValueError(cq, "No match set for cores ('coreA', 'coreB')")
     cq.addMatch(dict(core='coreA', uniqueKey='kA'), dict(core='coreB', key='kB'))
     self.assertEquals(['field=value'], cq.otherCoreFacetFiltersFor('coreB'))
     cq.convertWith(coreB=lambda q: "converted_" + q)
     self.assertEquals(['converted_field=value'], cq.otherCoreFacetFiltersFor('coreB'))
 def testAddDrilldownQuery(self):
     cq = ComposedQuery('coreA')
     cq.addDrilldownQuery('coreB', ('field', ['value']))
     self.assertValidateRaisesValueError(cq, "No match set for cores ('coreA', 'coreB')")
     cq.addMatch(dict(core='coreA', uniqueKey='kA'), dict(core='coreB', key='kB'))
     self.assertEquals([('field', ['value'])], cq.drilldownQueriesFor('coreB'))
     cq.convertWith(coreB=lambda q: "converted_" + q)
     self.assertEquals([('field', ['value'])], cq.drilldownQueriesFor('coreB'))
 def testAddRankQuery(self):
     cq = ComposedQuery('coreA')
     cq.setRankQuery('coreB', 'qB')
     self.assertValidateRaisesValueError(cq, "No match set for cores ('coreA', 'coreB')")
     cq.addMatch(dict(core='coreA', uniqueKey='kA'), dict(core='coreB', key='kB'))
     self.assertEquals('qB', cq.rankQueryFor('coreB'))
     cq.convertWith(coreB=lambda q: "converted_" + q)
     self.assertEquals('converted_qB', cq.rankQueryFor('coreB'))
Пример #21
0
 def testAddOtherCoreFacetFilter(self):
     cq = ComposedQuery('coreA')
     cq.addOtherCoreFacetFilter('coreB', 'field=value')
     self.assertValidateRaisesValueError(
         cq, "No match set for cores ('coreA', 'coreB')")
     cq.addMatch(dict(core='coreA', uniqueKey='kA'),
                 dict(core='coreB', key='kB'))
     self.assertEquals(['field=value'],
                       cq.otherCoreFacetFiltersFor('coreB'))
     cq.convertWith(coreB=lambda q: "converted_" + q)
     self.assertEquals(['converted_field=value'],
                       cq.otherCoreFacetFiltersFor('coreB'))
Пример #22
0
 def testUniteResultFromTwoIndexes(self):
     q = ComposedQuery('coreA')
     q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'))
     q.setCoreQuery(core='coreB', query=None)
     q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
     q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
     result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
     self.assertEquals(3, result.total)
     self.assertEquals(set(['A-QU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
Пример #23
0
    def testUniteResultFromTwoIndexesCachedAfterUpdate(self):
        q = ComposedQuery('coreA')
        q.setCoreQuery(core='coreA', query=luceneQueryFromCql('Q=true'), facets=[
                dict(fieldname='cat_Q', maxTerms=10),
                dict(fieldname='cat_U', maxTerms=10),
            ])
        q.setCoreQuery(core='coreB', query=None, facets=[
                dict(fieldname='cat_N', maxTerms=10),
                dict(fieldname='cat_O', maxTerms=10),
            ])
        q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
        q.addUnite(dict(core='coreA', query=luceneQueryFromCql('U=true')), dict(core='coreB', query=luceneQueryFromCql('N=true')))
        resultOne = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
        self.assertEquals(3, resultOne.total)
        self.assertEquals([{
                'terms': [
                    {'count': 3, 'term': u'true'}
                ], 'path': [], 'fieldname': u'cat_Q'
            }, {
                'terms': [
                    {'count': 2, 'term': u'true'},
                    {'count': 1, 'term': u'false'}
                ], 'path': [], 'fieldname': u'cat_U'
            }, {
                'terms': [
                    {'count': 2, 'term': u'true'}
                ], 'path': [], 'fieldname': u'cat_N'
            }, {
                'terms': [
                    {'count': 1, 'term': u'true'},
                    {'count': 1, 'term': u'false'},
                ], 'path': [], 'fieldname': u'cat_O'
            }], resultOne.drilldownData)

        self.addDocument(self.luceneA, identifier='A-MQU',  keys=[('A', 8 )], fields=[('M', 'true' ), ('Q', 'false' ), ('U', 'true' ), ('S', '8')])

        resultAgain = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
        self.assertEquals(2, resultAgain.total)
        self.assertEquals([{
                'terms': [
                    {'count': 2, 'term': u'true'}
                ], 'path': [], 'fieldname': u'cat_Q'
            }, {
                'terms': [
                    {'count': 1, 'term': u'false'},
                    {'count': 1, 'term': u'true'},
                ], 'path': [], 'fieldname': u'cat_U'
            }, {
                'terms': [
                    {'count': 1, 'term': u'true'}
                ], 'path': [], 'fieldname': u'cat_N'
            }, {
                'terms': [
                    {'count': 1, 'term': u'true'}
                ], 'path': [], 'fieldname': u'cat_O'
            }], resultAgain.drilldownData)
Пример #24
0
 def testUnite(self):
     cq = ComposedQuery('coreA')
     cq.setCoreQuery(core='coreA', query=None)
     cq.setCoreQuery(core='coreB', query=None)
     cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
     cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery'))
     self.assertEquals([('coreA', 'AQuery'), ('coreB', 'anotherQuery')], cq.unites)
Пример #25
0
    def testRemoteExecuteQuery(self):
        http = CallTrace('http')

        def httppost(*args, **kwargs):
            raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' %
                                LuceneResponse(total=5,
                                               hits=[
                                                   Hit("1"),
                                                   Hit("2"),
                                                   Hit("3", duplicateCount=2),
                                                   Hit("4"),
                                                   Hit("5")
                                               ]).asJson())
            yield

        http.methods['httppost'] = httppost
        remote = LuceneRemote(host='host', port=1234, path='/path')
        observable = Observable()
        observable.addObserver(remote)
        remote._httppost = http.httppost

        cq = ComposedQuery('coreA')
        cq.setCoreQuery(
            core='coreA',
            query=parseString('query AND  field=value'),
            filterQueries=[parseString('query=fiets')],
            facets=[{
                'fieldname': 'field',
                'maxTerms': 5
            }],
        )
        cq.setCoreQuery(core='coreB', query=parseString('query=test'))
        cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
                    dict(core='coreB', key='keyB'))
        result = returnValueFromGenerator(
            observable.any.executeComposedQuery(query=cq))
        self.assertEquals(5, result.total)
        self.assertEquals([
            Hit("1"),
            Hit("2"),
            Hit("3", duplicateCount=2),
            Hit("4"),
            Hit("5")
        ], result.hits)

        self.assertEquals(['httppost'], http.calledMethodNames())
        m = http.calledMethods[0]
        self.assertEquals('host', m.kwargs['host'])
        self.assertEquals(1234, m.kwargs['port'])
        self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
        self.assertEquals('application/json',
                          m.kwargs['headers']['Content-Type'])
        message, kwargs = jsonLoadMessage(m.kwargs['body'])
        query = kwargs['query']
        self.assertEquals('executeComposedQuery', message)
        self.assertEquals('coreA', query.resultsFrom)
        self.assertEquals([{
            'fieldname': 'field',
            'maxTerms': 5
        }], query.facetsFor('coreA'))
Пример #26
0
 def testAtMostOneMultiCoreOr(self):
     composedQuery = ComposedQuery('coreA')
     composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'),
                            dict(core='coreB', key='keyB'))
     composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'),
                            dict(core='coreC', key='keyC'))
     composedQuery.addUnite(dict(core='coreA', query='qA'),
                            dict(core='coreB', query='qB'))
     self.assertRaises(
         ValueError, lambda: composedQuery.addUnite(
             dict(core='coreA', query='qA'), dict('coreC', query='qC')))
 def testConvertComposedQuery(self):
     q = ComposedQuery('A')
     q.setCoreQuery(core='A', query=cqlToExpression('valueAQ'))
     q.setCoreQuery(core='B', query=cqlToExpression('valueBQ'))
     q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B', key='keyB'))
     q.addUnite(dict(core='A', query=cqlToExpression('fieldUA exact valueUA')), dict(core='B', query=cqlToExpression('fieldUB exact valueUB')))
     q.validate()
     consume(self.dna.any.executeComposedQuery(query=q))
     self.assertEquals(['executeComposedQuery'], self.observer.calledMethodNames())
     self.assertEquals("{'type': 'TermQuery', 'term': {'field': 'fieldA', 'value': u'valueaq'}, 'boost': 1.0}", repr(q.queryFor('A')))
     self.assertEquals("{'type': 'TermQuery', 'term': {'field': 'fieldB', 'value': u'valuebq'}, 'boost': 1.0}", repr(q.queryFor('B')))
Пример #28
0
 def testKeyName(self):
     cq = ComposedQuery('coreA')
     cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
                 dict(core='coreB', key='keyB'))
     cq.addFacet(core='coreB', facet='F0')
     self.assertEquals('keyA', cq.keyName('coreA', 'coreB'))
     self.assertEquals('keyB', cq.keyName('coreB', 'coreA'))
     self.assertEquals(set(['keyA']), cq.keyNames('coreA'))
Пример #29
0
def _loads_object_hook(dct):
    if '__CQL_QUERY__' in dct:
        return parseString(dct['__CQL_QUERY__'])
    elif '__COMPOSED_QUERY__' in dct:
        return ComposedQuery.fromDict(
            loads(dct['__COMPOSED_QUERY__'], object_hook=_loads_object_hook))
    return dct
Пример #30
0
    def testScoreCollectorCacheInvalidation(self):
        q = ComposedQuery('coreA', query=MatchAllDocsQuery())
        q.setRankQuery(core='coreC', query=luceneQueryFromCql('S=true'))
        q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C'))
        result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
        self.assertEquals(8, result.total)
        self.assertEquals(u'A-MQU', result.hits[0].id)
        self.assertEquals(set([u'A', u'A-U', u'A-Q', u'A-QU', u'A-M', u'A-MU', u'A-MQ']), set([hit.id for hit in result.hits[1:]]))

        self.addDocument(self.luceneC, identifier='C-S>A-MQ', keys=[('C', 7)], fields=[('S', 'true')])
        try:
            result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
            self.assertEquals(8, result.total)
            self.assertEquals(set([u'A-MQ' , u'A-MQU']), set([hit.id for hit in result.hits[:2]]))
            self.assertEquals(set([u'A', u'A-U', u'A-Q', u'A-QU', u'A-M', u'A-MU']), set([hit.id for hit in result.hits[2:]]))
        finally:
            self.luceneC.delete(identifier='C-S>A-MQ')
    def testValidateComposedQuery(self):
        composedQuery = ComposedQuery('coreA')
        composedQuery.setCoreQuery(core='coreA', query='Q0')
        composedQuery.setCoreQuery(core='coreB', query='Q1')
        self.assertValidateRaisesValueError(composedQuery, "No match set for cores ('coreA', 'coreB')")

        composedQuery = ComposedQuery('coreA', query="A")
        composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
        composedQuery.validate()
        self.assertEquals(1, composedQuery.numberOfUsedCores)
Пример #32
0
 def testKeyName(self):
     composedQuery = ComposedQuery('coreA')
     composedQuery.setCoreQuery(core='coreA', query=None)
     composedQuery.setCoreQuery(core='coreB', query=None)
     composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
     self.assertEquals('keyA', composedQuery.keyName('coreA'))
     self.assertEquals('keyB', composedQuery.keyName('coreB'))
Пример #33
0
    def testAddTypeAndMissingValueToSortField(self):
        self.response = JsonDict({
                "total": 887,
                "queryTime": 6,
                "hits": [{"id": "record:1", "score": 0.1234}]
            }).dumps()

        cq = ComposedQuery('coreA')
        q = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
        cq.setCoreQuery('coreB', q)
        cq.sortKeys = [dict(sortBy='sortField', core='coreA', sortDescending=True)]
        cq.addMatch(dict(core='coreA', uniqueKey='A'), dict(core='coreB', key='B'))
        consume(self._multiLucene.executeComposedQuery(cq))
        self.assertEqual({
                "_sortKeys": [{'core': 'coreA', 'sortBy': 'sortField', 'sortDescending': True, 'type': 'String', 'missingValue': 'STRING_FIRST'}],
                "resultsFrom": "coreA",
                '_matches': {'coreA->coreB': [{'core': 'coreA', 'uniqueKey': 'A'}, {'core': 'coreB', 'key': 'B'}]},
                "_facets": {},
                "_otherCoreFacetFilters": {},
                "_rankQueries": {},
                "_drilldownQueries": {},
                "_unites": [],
                '_queries': {'coreB': {'term': {'field': 'field', 'value': 'value'}, 'type': 'TermQuery'}},
                "cores": ["coreB", "coreA"],
                "_filterQueries": {}
            }, loads(self.post[0]['data']))
Пример #34
0
 def testJoinFacetAndQuery(self):
     q = ComposedQuery('coreA')
     q.setCoreQuery(core='coreA', query=None)
     q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'), facets=[
             dict(fieldname='cat_N', maxTerms=10),
             dict(fieldname='cat_O', maxTerms=10),
         ])
     q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX + 'A'), dict(core='coreB', key=KEY_PREFIX + 'B'))
     result = returnValueFromGenerator(self.dna.any.executeComposedQuery(query=q))
     self.assertEquals(4, result.total)
     self.assertEquals(set(['A-M', 'A-MU', 'A-MQ', 'A-MQU']), self.hitIds(result.hits))
     self.assertEquals([{
             'terms': [
                 {'count': 4, 'term': u'true'},
             ],
             'path': [],
             'fieldname': u'cat_N'
         }, {
             'terms': [
                 {'count': 2, 'term': u'true'},
                 {'count': 2, 'term': u'false'},
             ],
             'path': [],
             'fieldname': u'cat_O'
         }], result.drilldownData)
    def testRemoteExecuteQuery(self):
        http = CallTrace('http')
        def httppost(*args, **kwargs):
            raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")]).asJson())
            yield
        http.methods['httppost'] = httppost
        remote = LuceneRemote(host='host', port=1234, path='/path')
        observable = Observable()
        observable.addObserver(remote)
        remote._httppost = http.httppost

        cq = ComposedQuery('coreA')
        cq.setCoreQuery(
                core='coreA',
                query=parseString('query AND  field=value'),
                filterQueries=[parseString('query=fiets')],
                facets=[{'fieldname': 'field', 'maxTerms':5}],
            )
        cq.setCoreQuery(core='coreB', query=parseString('query=test'))
        cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
        result = returnValueFromGenerator(observable.any.executeComposedQuery(query=cq))
        self.assertEquals(5, result.total)
        self.assertEquals([Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")], result.hits)

        self.assertEquals(['httppost'], http.calledMethodNames())
        m = http.calledMethods[0]
        self.assertEquals('host', m.kwargs['host'])
        self.assertEquals(1234, m.kwargs['port'])
        self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
        self.assertEquals('application/json', m.kwargs['headers']['Content-Type'])
        message, kwargs = Conversion().jsonLoadMessage(m.kwargs['body'])
        query = kwargs['query']
        self.assertEquals('executeComposedQuery', message)
        self.assertEquals('coreA', query.resultsFrom)
        self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], query.facetsFor('coreA'))
 def testOtherCoreTermQuery(self):
     query = ComposedQuery('thisCore')
     query.cores.add('otherCore')
     query.addMatch(
         dict(core='thisCore', uniqueKey='A'),
         dict(core='otherCore', uniqueKey='B')
     )
     self.assertEquals({
         "type": "RelationalLuceneQuery",  # should this not be 'joined' to own core somehow? (with MatchAllDocs)
         "core": "otherCore",
         "collectKeyName": "B",
         "filterKeyName": "B",
         "query": {
             "type": "TermQuery",
             "term": {
                 "field": "field",
                 "value": "value",
             }
         }}, self._convert(QueryExpression.searchterm("otherCore.field", "=", "value"), composedQuery=query))
Пример #37
0
 def testKeyFilterIgnoresKeysOutOfBoundsOfKeySet(self):
     self.addDocument(self.luceneB, identifier=str(100), keys=[('B', 100)], fields=[]) # Force key to be much more than bits in long[] in FixedBitSet, so it must be OutOfBounds
     q = ComposedQuery('coreA')
     q.setCoreQuery(core='coreA', query=MatchAllDocsQuery())
     q.setCoreQuery(core='coreB', query=MatchAllDocsQuery())
     q.addFacet(core='coreB', facet=dict(fieldname='cat_M', maxTerms=10))
     q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
     result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
     self.assertEquals(4, len(result.hits))
Пример #38
0
 def testRankQuery(self):
     q = ComposedQuery('coreA', query=MatchAllDocsQuery())
     q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'))
     q.setRankQuery(core='coreC', query=luceneQueryFromCql('S=true'))
     q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
     q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C'))
     result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
     self.assertEquals(4, result.total)
     self.assertEquals([u'A-MQU', 'A-M', 'A-MU', u'A-MQ'], [hit.id for hit in result.hits])
Пример #39
0
 def testJoinQueryThreeCores(self):
     q = ComposedQuery('coreA')
     q.setCoreQuery(core='coreB', query=luceneQueryFromCql('N=true'))
     q.setCoreQuery(core='coreC', query=luceneQueryFromCql('R=true'))
     q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreB', key=KEY_PREFIX+'B'))
     q.addMatch(dict(core='coreA', uniqueKey=KEY_PREFIX+'A'), dict(core='coreC', key=KEY_PREFIX+'C'))
     q.addFacet(core='coreA', facet=dict(fieldname='cat_M', maxTerms=10))
     q.addFacet(core='coreB', facet=dict(fieldname='cat_N', maxTerms=10))
     q.addFacet(core='coreC', facet=dict(fieldname='cat_R', maxTerms=10))
     result = returnValueFromGenerator(self.dna.any.executeComposedQuery(q))
     self.assertEquals(1, result.total)
     self.assertEquals(set(['A-M']), self.hitIds(result.hits))
     self.assertEquals([
            {'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_M'},
            {'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_N'},
            {'terms': [{'count': 1, 'term': u'true'}], 'path': [], 'fieldname': u'cat_R'},
         ], result.drilldownData)
Пример #40
0
 def testSameCoreInDifferentMatchesRequiredToHaveSameKeyForNow(self):
     composedQuery = ComposedQuery('coreA', query='qA')
     composedQuery.setCoreQuery('coreB', query='qB')
     composedQuery.setCoreQuery('coreC', query='qC')
     composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'),
                            dict(core='coreB', key='keyB'))
     self.assertRaises(
         ValueError,
         lambda: composedQuery.addMatch(dict(core='coreA', key='keyX'),
                                        dict(core='coreC', key='keyC')))
 def testUnite(self):
     cq = ComposedQuery('coreA')
     cq.setCoreQuery(core='coreA', query=None)
     cq.setCoreQuery(core='coreB', query=None)
     cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
     cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery'))
     self.assertEqual(1, len(cq.unites))
     queries = list(cq.unites[0].queries())
     self.assertEquals(({'query': 'AQuery', 'keyName': 'keyA', 'core': 'coreA'}, 'keyA'), queries[0])
     self.assertEquals(({'query': 'anotherQuery', 'keyName': 'keyB', 'core': 'coreB'}, 'keyA'), queries[1])
Пример #42
0
    def testAddTypeAndMissingValueToSortField(self):
        self.response = JsonDict({"total": 887, "queryTime": 6, "hits": [{"id": "record:1", "score": 0.1234}]}).dumps()

        cq = ComposedQuery("coreA")
        q = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
        cq.setCoreQuery("coreB", q)
        cq.sortKeys = [dict(sortBy="sortField", core="coreA", sortDescending=True)]
        cq.addMatch(dict(core="coreA", uniqueKey="A"), dict(core="coreB", key="B"))
        consume(self._multiLucene.executeComposedQuery(cq))
        self.assertEqual(
            {
                "_sortKeys": [
                    {
                        "core": "coreA",
                        "sortBy": "sortField",
                        "sortDescending": True,
                        "type": "String",
                        "missingValue": "STRING_FIRST",
                    }
                ],
                "resultsFrom": "coreA",
                "_matches": {"coreA->coreB": [{"core": "coreA", "uniqueKey": "A"}, {"core": "coreB", "key": "B"}]},
                "_facets": {},
                "_otherCoreFacetFilters": {},
                "_rankQueries": {},
                "_drilldownQueries": {},
                "_unites": [],
                "_queries": {"coreB": {"term": {"field": "field", "value": "value"}, "type": "TermQuery"}},
                "cores": ["coreB", "coreA"],
                "_filterQueries": {},
            },
            loads(self.post[0]["data"]),
        )
Пример #43
0
    def testInfoOnQuery(self):
        self.response = JsonDict({
                "total": 887,
                "queryTime": 6,
                "hits": [{"id": "record:1", "score": 0.1234}]
            }).dumps()

        q = ComposedQuery('coreA')
        q.addFilterQuery('coreB', query='N=true')
        q.addMatch(dict(core='coreA', uniqueKey='A'), dict(core='coreB', key='B'))
        result = retval(self._multiLucene.executeComposedQuery(q))
        self.assertEquals({
            'query': {
                'cores': ['coreB', 'coreA'],
                'drilldownQueries': {},
                'facets': {},
                'filterQueries': {'coreB': ['N=true']},
                'matches': {'coreA->coreB': [{'core': 'coreA', 'uniqueKey': 'A'}, {'core': 'coreB', 'key': 'B'}]},
                'otherCoreFacetFilters': {},
                'queries': {},
                'rankQueries': {},
                'resultsFrom': 'coreA',
                'sortKeys': [],
                'unites': []
            },
            'type': 'ComposedQuery'
        }, result.info)
 def testOtherCoreTermQuery(self):
     query = ComposedQuery('thisCore')
     query.cores.add('otherCore')
     query.addMatch(dict(core='thisCore', uniqueKey='A'),
                    dict(core='otherCore', uniqueKey='B'))
     self.assertEquals(
         {
             "type":
             "RelationalLuceneQuery",  # should this not be 'joined' to own core somehow? (with MatchAllDocs)
             "core": "otherCore",
             "collectKeyName": "B",
             "filterKeyName": "B",
             "query": {
                 "type": "TermQuery",
                 "term": {
                     "field": "field",
                     "value": "value",
                 }
             }
         },
         self._convert(QueryExpression.searchterm("otherCore.field", "=",
                                                  "value"),
                       composedQuery=query))
Пример #45
0
 def testLuceneServerHostPortDynamic(self):
     multiLucene = MultiLucene(defaultCore='core1')
     def httprequest1_1Mock(**kwargs):
         raise StopIteration(parseResponse(HTTP_RESPONSE))
         yield
     observer = CallTrace(
         'observer',
         returnValues=dict(luceneServer=('example.org', 1234)),
         methods=dict(httprequest1_1=httprequest1_1Mock))
     multiLucene.addObserver(observer)
     query = QueryExpressionToLuceneQueryDict([], LuceneSettings()).convert(cqlToExpression("field=value"))
     response = retval(multiLucene.executeComposedQuery(ComposedQuery('core1', query)))
     self.assertEquals(887, response.total)
     self.assertEquals(['luceneServer', 'httprequest1_1'], observer.calledMethodNames())
Пример #46
0
 def testValidateComposedQueryForInvalidJson(self):
     composedQuery = ComposedQuery('coreA', query='Q0')
     composedQuery.relationalFilterJson = 'not JSON'
     self.assertValidateRaisesValueError(
         composedQuery,
         "Value 'not JSON' for 'relationalFilterJson' can not be parsed as JSON."
     )
     composedQuery.relationalFilterJson = '{"type": "MockJoinQuery"}'
     composedQuery.validate()
Пример #47
0
 def testUnite(self):
     cq = ComposedQuery('coreA')
     cq.setCoreQuery(core='coreA', query=None)
     cq.setCoreQuery(core='coreB', query=None)
     cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
                 dict(core='coreB', key='keyB'))
     cq.addUnite(dict(core='coreA', query='AQuery'),
                 dict(core='coreB', query='anotherQuery'))
     self.assertEqual(1, len(cq.unites))
     queries = list(cq.unites[0].queries())
     self.assertEquals(({
         'query': 'AQuery',
         'keyName': 'keyA',
         'core': 'coreA'
     }, 'keyA'), queries[0])
     self.assertEquals(({
         'query': 'anotherQuery',
         'keyName': 'keyB',
         'core': 'coreB'
     }, 'keyA'), queries[1])
Пример #48
0
    def testAsDictFromDict(self):
        cq = ComposedQuery('coreA')
        cq.setCoreQuery(core='coreA', query='Q0')
        cq.addFilterQuery(core='coreA', query='Q1')
        cq.addFilterQuery(core='coreA', query='Q2')
        cq.addFacet(core='coreA', facet='F0')
        cq.addFacet(core='coreA', facet='F1')
        cq.setCoreQuery(core='coreB', query='Q3')
        cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
        cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery'))
        cq.start = 0
        cq.sortKeys = [dict(sortBy='field', sortDescending=True)]

        d = cq.asDict()
        cq2 = ComposedQuery.fromDict(d)
        self.assertEquals('coreA', cq2.resultsFrom)
        self.assertEquals(0, cq2.start)
        self.assertEquals(None, cq2.stop)
        self.assertEquals(['Q0', 'Q1', 'Q2'], cq2.queriesFor('coreA'))
        self.assertEquals(['F0', 'F1'], cq2.facetsFor('coreA'))
        self.assertEquals('keyA', cq2.keyName('coreA'))
        self.assertEquals('keyB', cq2.keyName('coreB'))
Пример #49
0
    def testAddFilterQueriesIncremental(self):
        cq = ComposedQuery('coreA')
        cq.addFilterQuery(core='coreA', query='Q1')
        cq.addFilterQuery(core='coreA', query='Q2')

        self.assertEquals(['Q1', 'Q2'], cq.filterQueriesFor('coreA'))
Пример #50
0
    def testConvertAllQueries(self):
        cq = ComposedQuery('coreA')
        cq.setCoreQuery(core='coreA', query='Q0')
        cq.addFilterQuery('coreA', 'Q1')
        cq.addFilterQuery('coreA', 'Q2')
        cq.setCoreQuery(core='coreB', query='Q3')
        cq.addFilterQuery('coreB', 'Q4')
        cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
        cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
        convertCoreA = lambda query: "Converted_A_{0}".format(query)
        convertCoreB = lambda query: "Converted_B_{0}".format(query)
        cq.convertWith(coreA=convertCoreA, coreB=convertCoreB)

        self.assertEquals("Converted_A_Q0", cq.queryFor('coreA'))
        self.assertEquals(["Converted_A_Q1", "Converted_A_Q2"], cq.filterQueriesFor('coreA'))
        self.assertEquals("Converted_B_Q3", cq.queryFor('coreB'))
        self.assertEquals(["Converted_B_Q4"], cq.filterQueriesFor('coreB'))
        self.assertEquals([('coreA', 'Converted_A_Q5'), ('coreB', 'Converted_B_Q6')], cq.unites)
Пример #51
0
 def testSingleCoreQuery(self):
     cq = ComposedQuery('coreA')
     cq.setCoreQuery(core='coreA', query='Q0')
     cq.validate()
     self.assertEquals(1, cq.numberOfUsedCores)
Пример #52
0
 def testUniteQueriesFor(self):
     cq = ComposedQuery('coreA')
     cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
     cq.addUnite(dict(core='coreA', query='Q5'), dict(core='coreB', query='Q6'))
     self.assertEquals(['Q5'], cq.uniteQueriesFor('coreA'))
     self.assertEquals(['Q6'], cq.uniteQueriesFor('coreB'))
Пример #53
0
 def testFilterQueries(self):
     cq = ComposedQuery('coreA')
     cq.setCoreQuery(core='coreA', query='Q0')
     cq.addFilterQuery(core='coreA', query='Q1')
     cq.addFilterQuery(core='coreA', query='Q2')
     cq.addFacet(core='coreA', facet='F0')
     cq.addFacet(core='coreA', facet='F1')
     cq.setCoreQuery(core='coreB', query='Q3')
     cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
     cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery'))
     self.assertEquals(None, cq.stop)
     self.assertEquals(None, cq.start)
     self.assertEquals(None, cq.sortKeys)
     cq.stop = 10
     cq.start = 0
     cq.sortKeys = [dict(sortBy='field', sortDescending=True)]
     self.assertEquals('Q0', cq.queryFor('coreA'))
     self.assertEquals(['Q1', 'Q2'], cq.filterQueriesFor('coreA'))
     self.assertEquals(['F0', 'F1'], cq.facetsFor('coreA'))
     self.assertEquals(10, cq.stop)
     self.assertEquals(0, cq.start)
     self.assertEquals([dict(sortBy='field', sortDescending=True)], cq.sortKeys)
Пример #54
0
 def testConvertComposedQuery(self):
     q = ComposedQuery('A')
     q.setCoreQuery(core='A', query=cqlToExpression('valueAQ'))
     q.setCoreQuery(core='B', query=cqlToExpression('valueBQ'))
     q.addMatch(dict(core='A', uniqueKey='keyA'), dict(core='B',
                                                       key='keyB'))
     q.addUnite(
         dict(core='A', query=cqlToExpression('fieldUA exact valueUA')),
         dict(core='B', query=cqlToExpression('fieldUB exact valueUB')))
     q.validate()
     consume(self.dna.any.executeComposedQuery(query=q))
     self.assertEquals(['executeComposedQuery'],
                       self.observer.calledMethodNames())
     self.assertEquals(
         "{'type': 'TermQuery', 'term': {'field': 'fieldA', 'value': u'valueaq'}, 'boost': 1.0}",
         repr(q.queryFor('A')))
     self.assertEquals(
         "{'type': 'TermQuery', 'term': {'field': 'fieldB', 'value': u'valuebq'}, 'boost': 1.0}",
         repr(q.queryFor('B')))
Пример #55
0
 def testUniqueKeyDoesntMatchResultsFrom(self):
     composedQuery = ComposedQuery('coreA', query='A').setCoreQuery('coreB', query='bQ')
     self.assertRaises(ValueError, lambda: composedQuery.addMatch(dict(core='coreA', key='keyA'), dict(core='coreB', key='keyB')))
     self.assertRaises(ValueError, lambda: composedQuery.addMatch(dict(core='coreA', key='keyA'), dict(core='coreB', uniqueKey='keyB')))
     composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
     composedQuery.validate()
     composedQuery.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', uniqueKey='keyB'))
     composedQuery.validate()