def testExportKeys(self): composedQuery = ComposedQuery('main') composedQuery.setCoreQuery('main', query=dict(type="MatchAllDocsQuery")) header, body = postRequest(self.luceneServerPort, '/exportkeys/?exportKey=__key__.field', data=JsonDict(composedQuery.asDict()).dumps(), parse=False) self.assertTrue("200 OK" in header.upper(), header + 2 * CRLF + body) bitSet = readFixedBitSet(body) for i in xrange(0, 102): isSet = bitSet.get(i) if 2 < i < 101: self.assertTrue(isSet, i) else: self.assertFalse(isSet, i)
def testExportKeys(self): composedQuery = ComposedQuery('main') composedQuery.setCoreQuery('main', query=dict(type="MatchAllDocsQuery")) header, body = postRequest(self.luceneServerPort, '/exportkeys/?exportKey=__key__.field', data=JsonDict(composedQuery.asDict()).dumps(), parse=False) self.assertTrue("200 OK" in header.upper(), header + 2 * CRLF + body) bitSet = readOpenBitSet(body) for i in xrange(0, 102): isSet = bitSet.get(i) if 2 < i < 101: self.assertTrue(isSet, i) else: self.assertFalse(isSet, i)
def testConversionOfComposedQuery(self): conversion = Conversion() cq = ComposedQuery('coreA') cq.setCoreQuery(core='coreA', query=parseString('Q0'), filterQueries=['Q1', 'Q2'], facets=['F0', 'F1']) cq.setCoreQuery(core='coreB', query=QueryExpression.searchterm(term='Q3'), filterQueries=['Q4']) cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB')) cq.addUnite(dict(core='coreA', query='AQuery'), dict(core='coreB', query='anotherQuery')) cq.start = 0 cq.sortKeys = [dict(sortBy='field', sortDescending=True)] kwargs = {'q': cq} dump = conversion.jsonDumpMessage(message='aMessage', **kwargs) self.assertEquals(str, type(dump)) message, kwargs = conversion.jsonLoadMessage(dump) self.assertEquals('aMessage', message) cq2 = kwargs['q'] self.assertEquals(parseString('Q0'), cq2.queryFor('coreA'))
def executeQuery(self, query=None, extraArguments=None, facets=None, drilldownQueries=None, filterQueries=None, sortKeys=None, **kwargs): if 'cqlAbstractSyntaxTree' in kwargs: query = kwargs.pop('cqlAbstractSyntaxTree') query = cqlToExpression(query) extraArguments = extraArguments or {} cq = ComposedQuery(self._resultsFrom) for matchTuple in self._matches: cq.addMatch(*matchTuple) for key in ['start', 'stop', 'suggestionRequest']: if key in kwargs: setattr(cq, key, kwargs[key]) coreQuery, filters = self._extraFilterQueries.convert(query, self._resultsFrom) if coreQuery: cq.setCoreQuery(core=self._resultsFrom, query=coreQuery) for core, aFilter in ((core, aFilter) for core, filters in filters.items() for aFilter in filters): cq.addFilterQuery(core, aFilter) for sortKey in sortKeys or []: core, sortBy = self._parseCorePrefix(sortKey['sortBy'], self._cores) cq.addSortKey(dict(sortKey, core=core, sortBy=sortBy)) filters = extraArguments.get('x-filter', []) for f in filters: core, filterQuery = self._coreQuery(query=f, cores=self._cores) cq.addFilterQuery(core=core, query=filterQuery) for core, filterQuery in filterQueries or []: cq.addFilterQuery(core=core, query=cqlToExpression(filterQuery)) rankQueries = extraArguments.get('x-rank-query', []) if rankQueries: queries = defaultdict(list) for rankQuery in rankQueries: core, rankQuery = self._parseCorePrefix(rankQuery, self._cores) queries[core].append(rankQuery) for core, q in queries.items(): cq.setRankQuery(core=core, query=cqlToExpression(' OR '.join(q))) if self._dedupFieldName: if 'true' == extraArguments.get('x-filter-common-keys', ['true' if self._dedupByDefault else 'false'])[0]: setattr(cq, "dedupField", self._dedupFieldName) setattr(cq, "dedupSortField", self._dedupSortFieldName) if self._clusteringEnabled and 'true' == extraArguments.get('x-clustering', [None])[0]: setattr(cq, "clustering", True) facetOrder = [] fieldTranslations = {} for drilldownField in (facets or []): path = drilldownField['fieldname'].split('>') fieldname, path = path[0], path[1:] facetOrder.append((fieldname, path)) core, newFieldname = self._coreFacet(fieldname, self._cores) newFieldname = self._drilldownFieldnamesTranslate(newFieldname) fieldTranslations[newFieldname] = fieldname cq.addFacet(core=core, facet=dict(fieldname=newFieldname, path=path, maxTerms=drilldownField['maxTerms'])) for drilldownQuery in (drilldownQueries or []): core, fieldname = self._coreDrilldownQuery(drilldownQuery[0], self._cores) fieldname = self._drilldownFieldnamesTranslate(fieldname) cq.addDrilldownQuery(core=core, drilldownQuery=(fieldname, drilldownQuery[1])) result = yield self.any.executeComposedQuery(query=cq) drilldownData = getattr(result, "drilldownData", None) if drilldownData: for facet in drilldownData: fieldname = facet['fieldname'] facet['fieldname'] = fieldTranslations.get(fieldname, fieldname) result.drilldownData = sorted( drilldownData, key=lambda d: facetOrder.index((d['fieldname'], d.get('path', []))) ) raise StopIteration(result)
def executeQuery(self, query=None, extraArguments=None, facets=None, drilldownQueries=None, filterQueries=None, sortKeys=None, **kwargs): if 'cqlAbstractSyntaxTree' in kwargs: query = kwargs.pop('cqlAbstractSyntaxTree') query = cqlToExpression(query) extraArguments = extraArguments or {} cq = ComposedQuery(self._resultsFrom) for matchTuple in self._matches: cq.addMatch(*matchTuple) for key in ['start', 'stop', 'suggestionRequest']: if key in kwargs: setattr(cq, key, kwargs[key]) coreQuery, filters = self._extraFilterQueries.convert( query, self._resultsFrom) if coreQuery: cq.setCoreQuery(core=self._resultsFrom, query=coreQuery) for core, aFilter in ((core, aFilter) for core, filters in filters.items() for aFilter in filters): cq.addFilterQuery(core, aFilter) for sortKey in sortKeys or []: core, sortBy = self._parseCorePrefix(sortKey['sortBy'], self._cores) cq.addSortKey(dict(sortKey, core=core, sortBy=sortBy)) filters = extraArguments.get('x-filter', []) for f in filters: core, filterQuery = self._coreQuery(query=f, cores=self._cores) cq.addFilterQuery(core=core, query=filterQuery) for core, filterQuery in filterQueries or []: cq.addFilterQuery(core=core, query=cqlToExpression(filterQuery)) rankQueries = extraArguments.get('x-rank-query', []) if rankQueries: queries = defaultdict(list) for rankQuery in rankQueries: core, rankQuery = self._parseCorePrefix(rankQuery, self._cores) queries[core].append(rankQuery) for core, q in queries.items(): cq.setRankQuery(core=core, query=cqlToExpression(' OR '.join(q))) filterCommonKeysField = extraArguments.get( 'x-filter-common-keys-field', [self._dedupFieldName])[0] if filterCommonKeysField and 'true' == extraArguments.get( 'x-filter-common-keys', ['true' if self._dedupByDefault else 'false'])[0]: cq.dedupField = ('' if filterCommonKeysField.startswith(KEY_PREFIX) else KEY_PREFIX) + filterCommonKeysField cq.dedupSortField = self._dedupSortFieldName if self._clusteringEnabled and 'true' == extraArguments.get( 'x-clustering', [None])[0]: setattr(cq, "clustering", True) facetOrder = [] fieldTranslations = {} for drilldownField in (facets or []): path = drilldownField['fieldname'].split('>') fieldname, path = path[0], path[1:] facetOrder.append((fieldname, path)) core, newFieldname = self._coreFacet(fieldname, self._cores) newFieldname = self._drilldownFieldnamesTranslate(newFieldname) fieldTranslations[newFieldname] = fieldname cq.addFacet(core=core, facet=dict(fieldname=newFieldname, path=path, maxTerms=drilldownField['maxTerms'])) for drilldownQuery in (drilldownQueries or []): core, fieldname = self._coreDrilldownQuery(drilldownQuery[0], self._cores) fieldname = self._drilldownFieldnamesTranslate(fieldname) cq.addDrilldownQuery(core=core, drilldownQuery=(fieldname, drilldownQuery[1])) result = yield self.any.executeComposedQuery(query=cq) drilldownData = getattr(result, "drilldownData", None) if drilldownData: for facet in drilldownData: fieldname = facet['fieldname'] facet['fieldname'] = fieldTranslations.get( fieldname, fieldname) result.drilldownData = sorted( drilldownData, key=lambda d: facetOrder.index( (d['fieldname'], d.get('path', [])))) raise StopIteration(result)