def testJoin(self):
remote = SynchronousRemote(host='localhost', port=self.httpPort, path='/remote')
q = ComposedQuery('main', query=cqlToExpression('*'))
q.addMatch(dict(core='main', uniqueKey=KEY_PREFIX+'field'), dict(core='main2', key=KEY_PREFIX+'field'))
q.start=0
q.stop=100
q.addFilterQuery(core='main', query=cqlToExpression('field2=value0 OR field2=value1'))
q.addFacet(core='main2', facet=dict(fieldname='untokenized.field2', maxTerms=5))
response = remote.executeComposedQuery(query=q)
self.assertEquals(19, response.total)
self.assertEquals(set([
'record:10', 'record:11', 'record:20', 'record:21', 'record:30',
'record:31', 'record:40', 'record:41', 'record:50', 'record:51',
'record:60', 'record:61', 'record:70', 'record:71', 'record:80',
'record:81', 'record:90', 'record:91', 'record:100'
]), set([hit.id for hit in response.hits]))
self.assertEquals([{
'fieldname': 'untokenized.field2',
'path': [],
'terms': [
{'count': 27, 'term': 'value3'},
{'count': 22, 'term': 'value0'},
{'count': 19, 'term': 'value5'},
{'count': 19, 'term': 'value7'},
{'count': 19, 'term': 'value9'},
]
}], response.drilldownData)
def testJoinWithSortAndMissingValue(self):
remote = SynchronousRemote(host='localhost', port=self.httpPort, path='/remote')
q = ComposedQuery('main', query=cqlToExpression('*'))
q.addMatch(dict(core='main', uniqueKey=KEY_PREFIX+'field'), dict(core='main2', key=KEY_PREFIX+'field'))
q.addFacet(core='main2', facet=dict(fieldname='untokenized.field2', maxTerms=5))
q.addSortKey(dict(core="main", sortBy="sorted.field4", sortDescending=True))
response = remote.executeComposedQuery(query=q)
self.assertEqual("record:1", response.hits[0].id)
del q._sortKeys[:]
q.addSortKey(dict(core="main", sortBy="sorted.field4", sortDescending=False))
response = remote.executeComposedQuery(query=q)
self.assertEqual("record:1", response.hits[0].id)
def testExportKeys(self):
composedQuery = ComposedQuery('main')
composedQuery.setCoreQuery('main', query=dict(type="MatchAllDocsQuery"))
header, body = postRequest(self.luceneServerPort, '/exportkeys/?exportKey=__key__.field', data=JsonDict(composedQuery.asDict()).dumps(), parse=False)
self.assertTrue("200 OK" in header.upper(), header + 2 * CRLF + body)
bitSet = readFixedBitSet(body)
for i in xrange(0, 102):
isSet = bitSet.get(i)
if 2 < i < 101:
self.assertTrue(isSet, i)
else:
self.assertFalse(isSet, i)
def testExportKeys(self):
composedQuery = ComposedQuery('main')
composedQuery.setCoreQuery('main', query=dict(type="MatchAllDocsQuery"))
header, body = postRequest(self.luceneServerPort, '/exportkeys/?exportKey=__key__.field', data=JsonDict(composedQuery.asDict()).dumps(), parse=False)
self.assertTrue("200 OK" in header.upper(), header + 2 * CRLF + body)
bitSet = readOpenBitSet(body)
for i in xrange(0, 102):
isSet = bitSet.get(i)
if 2 < i < 101:
self.assertTrue(isSet, i)
else:
self.assertFalse(isSet, i)
def executeQuery(self, query=None, extraArguments=None, facets=None, drilldownQueries=None, filterQueries=None, sortKeys=None, **kwargs):
if 'cqlAbstractSyntaxTree' in kwargs:
query = kwargs.pop('cqlAbstractSyntaxTree')
query = cqlToExpression(query)
extraArguments = extraArguments or {}
cq = ComposedQuery(self._resultsFrom)
for matchTuple in self._matches:
cq.addMatch(*matchTuple)
for key in ['start', 'stop', 'suggestionRequest']:
if key in kwargs:
setattr(cq, key, kwargs[key])
coreQuery, filters = self._extraFilterQueries.convert(query, self._resultsFrom)
if coreQuery:
cq.setCoreQuery(core=self._resultsFrom, query=coreQuery)
for core, aFilter in ((core, aFilter) for core, filters in filters.items() for aFilter in filters):
cq.addFilterQuery(core, aFilter)
for sortKey in sortKeys or []:
core, sortBy = self._parseCorePrefix(sortKey['sortBy'], self._cores)
cq.addSortKey(dict(sortKey, core=core, sortBy=sortBy))
filters = extraArguments.get('x-filter', [])
for f in filters:
core, filterQuery = self._coreQuery(query=f, cores=self._cores)
cq.addFilterQuery(core=core, query=filterQuery)
for core, filterQuery in filterQueries or []:
cq.addFilterQuery(core=core, query=cqlToExpression(filterQuery))
rankQueries = extraArguments.get('x-rank-query', [])
if rankQueries:
queries = defaultdict(list)
for rankQuery in rankQueries:
core, rankQuery = self._parseCorePrefix(rankQuery, self._cores)
queries[core].append(rankQuery)
for core, q in queries.items():
cq.setRankQuery(core=core, query=cqlToExpression(' OR '.join(q)))
if self._dedupFieldName:
if 'true' == extraArguments.get('x-filter-common-keys', ['true' if self._dedupByDefault else 'false'])[0]:
setattr(cq, "dedupField", self._dedupFieldName)
setattr(cq, "dedupSortField", self._dedupSortFieldName)
if self._clusteringEnabled and 'true' == extraArguments.get('x-clustering', [None])[0]:
setattr(cq, "clustering", True)
facetOrder = []
fieldTranslations = {}
for drilldownField in (facets or []):
path = drilldownField['fieldname'].split('>')
fieldname, path = path[0], path[1:]
facetOrder.append((fieldname, path))
core, newFieldname = self._coreFacet(fieldname, self._cores)
newFieldname = self._drilldownFieldnamesTranslate(newFieldname)
fieldTranslations[newFieldname] = fieldname
cq.addFacet(core=core, facet=dict(fieldname=newFieldname, path=path, maxTerms=drilldownField['maxTerms']))
for drilldownQuery in (drilldownQueries or []):
core, fieldname = self._coreDrilldownQuery(drilldownQuery[0], self._cores)
fieldname = self._drilldownFieldnamesTranslate(fieldname)
cq.addDrilldownQuery(core=core, drilldownQuery=(fieldname, drilldownQuery[1]))
result = yield self.any.executeComposedQuery(query=cq)
drilldownData = getattr(result, "drilldownData", None)
if drilldownData:
for facet in drilldownData:
fieldname = facet['fieldname']
facet['fieldname'] = fieldTranslations.get(fieldname, fieldname)
result.drilldownData = sorted(
drilldownData,
key=lambda d: facetOrder.index((d['fieldname'], d.get('path', [])))
)
raise StopIteration(result)
def testConversionOfComposedQuery(self):
conversion = Conversion()
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA', query=parseString('Q0'), filterQueries=['Q1', 'Q2'], facets=['F0', 'F1'])
cq.setCoreQuery(core='coreB', query=QueryExpression.searchterm(term='Q3'), filterQueries=['Q4'])
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery'))
cq.start = 0
cq.sortKeys = [dict(sortBy='field', sortDescending=True)]
kwargs = {'q': cq}
dump = conversion.jsonDumpMessage(message='aMessage', **kwargs)
self.assertEquals(str, type(dump))
message, kwargs = conversion.jsonLoadMessage(dump)
self.assertEquals('aMessage', message)
cq2 = kwargs['q']
self.assertEquals(parseString('Q0'), cq2.queryFor('coreA'))
def testConversionOfComposedQuery(self):
conversion = Conversion()
cq = ComposedQuery('coreA')
cq.setCoreQuery(core='coreA',
query=parseString('Q0'),
filterQueries=['Q1', 'Q2'],
facets=['F0', 'F1'])
cq.setCoreQuery(core='coreB',
query=QueryExpression.searchterm(term='Q3'),
filterQueries=['Q4'])
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
cq.addUnite(dict(core='coreA', query='AQuery'),
dict(core='coreB', query='anotherQuery'))
cq.start = 0
cq.sortKeys = [dict(sortBy='field', sortDescending=True)]
kwargs = {'q': cq}
dump = conversion.jsonDumpMessage(message='aMessage', **kwargs)
self.assertEquals(str, type(dump))
message, kwargs = conversion.jsonLoadMessage(dump)
self.assertEquals('aMessage', message)
cq2 = kwargs['q']
self.assertEquals(parseString('Q0'), cq2.queryFor('coreA'))
def executeQuery(self,
query=None,
extraArguments=None,
facets=None,
drilldownQueries=None,
filterQueries=None,
sortKeys=None,
**kwargs):
if 'cqlAbstractSyntaxTree' in kwargs:
query = kwargs.pop('cqlAbstractSyntaxTree')
query = cqlToExpression(query)
extraArguments = extraArguments or {}
cq = ComposedQuery(self._resultsFrom)
for matchTuple in self._matches:
cq.addMatch(*matchTuple)
for key in ['start', 'stop', 'suggestionRequest']:
if key in kwargs:
setattr(cq, key, kwargs[key])
coreQuery, filters = self._extraFilterQueries.convert(
query, self._resultsFrom)
if coreQuery:
cq.setCoreQuery(core=self._resultsFrom, query=coreQuery)
for core, aFilter in ((core, aFilter)
for core, filters in filters.items()
for aFilter in filters):
cq.addFilterQuery(core, aFilter)
for sortKey in sortKeys or []:
core, sortBy = self._parseCorePrefix(sortKey['sortBy'],
self._cores)
cq.addSortKey(dict(sortKey, core=core, sortBy=sortBy))
filters = extraArguments.get('x-filter', [])
for f in filters:
core, filterQuery = self._coreQuery(query=f, cores=self._cores)
cq.addFilterQuery(core=core, query=filterQuery)
for core, filterQuery in filterQueries or []:
cq.addFilterQuery(core=core, query=cqlToExpression(filterQuery))
rankQueries = extraArguments.get('x-rank-query', [])
if rankQueries:
queries = defaultdict(list)
for rankQuery in rankQueries:
core, rankQuery = self._parseCorePrefix(rankQuery, self._cores)
queries[core].append(rankQuery)
for core, q in queries.items():
cq.setRankQuery(core=core,
query=cqlToExpression(' OR '.join(q)))
filterCommonKeysField = extraArguments.get(
'x-filter-common-keys-field', [self._dedupFieldName])[0]
if filterCommonKeysField and 'true' == extraArguments.get(
'x-filter-common-keys',
['true' if self._dedupByDefault else 'false'])[0]:
cq.dedupField = ('' if filterCommonKeysField.startswith(KEY_PREFIX)
else KEY_PREFIX) + filterCommonKeysField
cq.dedupSortField = self._dedupSortFieldName
if self._clusteringEnabled and 'true' == extraArguments.get(
'x-clustering', [None])[0]:
setattr(cq, "clustering", True)
facetOrder = []
fieldTranslations = {}
for drilldownField in (facets or []):
path = drilldownField['fieldname'].split('>')
fieldname, path = path[0], path[1:]
facetOrder.append((fieldname, path))
core, newFieldname = self._coreFacet(fieldname, self._cores)
newFieldname = self._drilldownFieldnamesTranslate(newFieldname)
fieldTranslations[newFieldname] = fieldname
cq.addFacet(core=core,
facet=dict(fieldname=newFieldname,
path=path,
maxTerms=drilldownField['maxTerms']))
for drilldownQuery in (drilldownQueries or []):
core, fieldname = self._coreDrilldownQuery(drilldownQuery[0],
self._cores)
fieldname = self._drilldownFieldnamesTranslate(fieldname)
cq.addDrilldownQuery(core=core,
drilldownQuery=(fieldname, drilldownQuery[1]))
result = yield self.any.executeComposedQuery(query=cq)
drilldownData = getattr(result, "drilldownData", None)
if drilldownData:
for facet in drilldownData:
fieldname = facet['fieldname']
facet['fieldname'] = fieldTranslations.get(
fieldname, fieldname)
result.drilldownData = sorted(
drilldownData,
key=lambda d: facetOrder.index(
(d['fieldname'], d.get('path', []))))
raise StopIteration(result)
