def testJoin(self): remote = SynchronousRemote(host='localhost', port=self.httpPort, path='/remote') q = ComposedQuery('main', query=cqlToExpression('*')) q.addMatch(dict(core='main', uniqueKey=KEY_PREFIX+'field'), dict(core='main2', key=KEY_PREFIX+'field')) q.start=0 q.stop=100 q.addFilterQuery(core='main', query=cqlToExpression('field2=value0 OR field2=value1')) q.addFacet(core='main2', facet=dict(fieldname='untokenized.field2', maxTerms=5)) response = remote.executeComposedQuery(query=q) self.assertEquals(19, response.total) self.assertEquals(set([ 'record:10', 'record:11', 'record:20', 'record:21', 'record:30', 'record:31', 'record:40', 'record:41', 'record:50', 'record:51', 'record:60', 'record:61', 'record:70', 'record:71', 'record:80', 'record:81', 'record:90', 'record:91', 'record:100' ]), set([hit.id for hit in response.hits])) self.assertEquals([{ 'fieldname': 'untokenized.field2', 'path': [], 'terms': [ {'count': 27, 'term': 'value3'}, {'count': 22, 'term': 'value0'}, {'count': 19, 'term': 'value5'}, {'count': 19, 'term': 'value7'}, {'count': 19, 'term': 'value9'}, ] }], response.drilldownData)
def testJoinWithSortAndMissingValue(self): remote = SynchronousRemote(host='localhost', port=self.httpPort, path='/remote') q = ComposedQuery('main', query=cqlToExpression('*')) q.addMatch(dict(core='main', uniqueKey=KEY_PREFIX+'field'), dict(core='main2', key=KEY_PREFIX+'field')) q.addFacet(core='main2', facet=dict(fieldname='untokenized.field2', maxTerms=5)) q.addSortKey(dict(core="main", sortBy="sorted.field4", sortDescending=True)) response = remote.executeComposedQuery(query=q) self.assertEqual("record:1", response.hits[0].id) del q._sortKeys[:] q.addSortKey(dict(core="main", sortBy="sorted.field4", sortDescending=False)) response = remote.executeComposedQuery(query=q) self.assertEqual("record:1", response.hits[0].id)
def testConversionOfComposedQuery(self): conversion = Conversion() cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=parseString('Q0'), filterQueries=['Q1', 'Q2'], facets=['F0', 'F1']) cq.setCoreQuery(core='coreB', query=QueryExpression.searchterm(term='Q3'), filterQueries=['Q4']) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery')) cq.start = 0 cq.sortKeys = [dict(sortBy='field', sortDescending=True)] kwargs = {'q': cq} dump = conversion.jsonDumpMessage(message='aMessage', **kwargs) self.assertEquals(str, type(dump)) message, kwargs = conversion.jsonLoadMessage(dump) self.assertEquals('aMessage', message) cq2 = kwargs['q'] self.assertEquals(parseString('Q0'), cq2.queryFor('coreA'))
def executeQuery(self, query=None, extraArguments=None, facets=None, drilldownQueries=None, filterQueries=None, sortKeys=None, **kwargs): if 'cqlAbstractSyntaxTree' in kwargs: query = kwargs.pop('cqlAbstractSyntaxTree') query = cqlToExpression(query) extraArguments = extraArguments or {} cq = ComposedQuery(self._resultsFrom) for matchTuple in self._matches: cq.addMatch(*matchTuple) for key in ['start', 'stop', 'suggestionRequest']: if key in kwargs: setattr(cq, key, kwargs[key]) coreQuery, filters = self._extraFilterQueries.convert(query, self._resultsFrom) if coreQuery: cq.setCoreQuery(core=self._resultsFrom, query=coreQuery) for core, aFilter in ((core, aFilter) for core, filters in filters.items() for aFilter in filters): cq.addFilterQuery(core, aFilter) for sortKey in sortKeys or []: core, sortBy = self._parseCorePrefix(sortKey['sortBy'], self._cores) cq.addSortKey(dict(sortKey, core=core, sortBy=sortBy)) filters = extraArguments.get('x-filter', []) for f in filters: core, filterQuery = self._coreQuery(query=f, cores=self._cores) cq.addFilterQuery(core=core, query=filterQuery) for core, filterQuery in filterQueries or []: cq.addFilterQuery(core=core, query=cqlToExpression(filterQuery)) rankQueries = extraArguments.get('x-rank-query', []) if rankQueries: queries = defaultdict(list) for rankQuery in rankQueries: core, rankQuery = self._parseCorePrefix(rankQuery, self._cores) queries[core].append(rankQuery) for core, q in queries.items(): cq.setRankQuery(core=core, query=cqlToExpression(' OR '.join(q))) if self._dedupFieldName: if 'true' == extraArguments.get('x-filter-common-keys', ['true' if self._dedupByDefault else 'false'])[0]: setattr(cq, "dedupField", self._dedupFieldName) setattr(cq, "dedupSortField", self._dedupSortFieldName) if self._clusteringEnabled and 'true' == extraArguments.get('x-clustering', [None])[0]: setattr(cq, "clustering", True) facetOrder = [] fieldTranslations = {} for drilldownField in (facets or []): path = drilldownField['fieldname'].split('>') fieldname, path = path[0], path[1:] facetOrder.append((fieldname, path)) core, newFieldname = self._coreFacet(fieldname, self._cores) newFieldname = self._drilldownFieldnamesTranslate(newFieldname) fieldTranslations[newFieldname] = fieldname cq.addFacet(core=core, facet=dict(fieldname=newFieldname, path=path, maxTerms=drilldownField['maxTerms'])) for drilldownQuery in (drilldownQueries or []): core, fieldname = self._coreDrilldownQuery(drilldownQuery[0], self._cores) fieldname = self._drilldownFieldnamesTranslate(fieldname) cq.addDrilldownQuery(core=core, drilldownQuery=(fieldname, drilldownQuery[1])) result = yield self.any.executeComposedQuery(query=cq) drilldownData = getattr(result, "drilldownData", None) if drilldownData: for facet in drilldownData: fieldname = facet['fieldname'] facet['fieldname'] = fieldTranslations.get(fieldname, fieldname) result.drilldownData = sorted( drilldownData, key=lambda d: facetOrder.index((d['fieldname'], d.get('path', []))) ) raise StopIteration(result)
def executeQuery(self, query=None, extraArguments=None, facets=None, drilldownQueries=None, filterQueries=None, sortKeys=None, **kwargs): if 'cqlAbstractSyntaxTree' in kwargs: query = kwargs.pop('cqlAbstractSyntaxTree') query = cqlToExpression(query) extraArguments = extraArguments or {} cq = ComposedQuery(self._resultsFrom) for matchTuple in self._matches: cq.addMatch(*matchTuple) for key in ['start', 'stop', 'suggestionRequest']: if key in kwargs: setattr(cq, key, kwargs[key]) coreQuery, filters = self._extraFilterQueries.convert( query, self._resultsFrom) if coreQuery: cq.setCoreQuery(core=self._resultsFrom, query=coreQuery) for core, aFilter in ((core, aFilter) for core, filters in filters.items() for aFilter in filters): cq.addFilterQuery(core, aFilter) for sortKey in sortKeys or []: core, sortBy = self._parseCorePrefix(sortKey['sortBy'], self._cores) cq.addSortKey(dict(sortKey, core=core, sortBy=sortBy)) filters = extraArguments.get('x-filter', []) for f in filters: core, filterQuery = self._coreQuery(query=f, cores=self._cores) cq.addFilterQuery(core=core, query=filterQuery) for core, filterQuery in filterQueries or []: cq.addFilterQuery(core=core, query=cqlToExpression(filterQuery)) rankQueries = extraArguments.get('x-rank-query', []) if rankQueries: queries = defaultdict(list) for rankQuery in rankQueries: core, rankQuery = self._parseCorePrefix(rankQuery, self._cores) queries[core].append(rankQuery) for core, q in queries.items(): cq.setRankQuery(core=core, query=cqlToExpression(' OR '.join(q))) filterCommonKeysField = extraArguments.get( 'x-filter-common-keys-field', [self._dedupFieldName])[0] if filterCommonKeysField and 'true' == extraArguments.get( 'x-filter-common-keys', ['true' if self._dedupByDefault else 'false'])[0]: cq.dedupField = ('' if filterCommonKeysField.startswith(KEY_PREFIX) else KEY_PREFIX) + filterCommonKeysField cq.dedupSortField = self._dedupSortFieldName if self._clusteringEnabled and 'true' == extraArguments.get( 'x-clustering', [None])[0]: setattr(cq, "clustering", True) facetOrder = [] fieldTranslations = {} for drilldownField in (facets or []): path = drilldownField['fieldname'].split('>') fieldname, path = path[0], path[1:] facetOrder.append((fieldname, path)) core, newFieldname = self._coreFacet(fieldname, self._cores) newFieldname = self._drilldownFieldnamesTranslate(newFieldname) fieldTranslations[newFieldname] = fieldname cq.addFacet(core=core, facet=dict(fieldname=newFieldname, path=path, maxTerms=drilldownField['maxTerms'])) for drilldownQuery in (drilldownQueries or []): core, fieldname = self._coreDrilldownQuery(drilldownQuery[0], self._cores) fieldname = self._drilldownFieldnamesTranslate(fieldname) cq.addDrilldownQuery(core=core, drilldownQuery=(fieldname, drilldownQuery[1])) result = yield self.any.executeComposedQuery(query=cq) drilldownData = getattr(result, "drilldownData", None) if drilldownData: for facet in drilldownData: fieldname = facet['fieldname'] facet['fieldname'] = fieldTranslations.get( fieldname, fieldname) result.drilldownData = sorted( drilldownData, key=lambda d: facetOrder.index( (d['fieldname'], d.get('path', [])))) raise StopIteration(result)