def __init__(self,
resultsFrom,
matches=None,
dedupFieldName=None,
dedupSortFieldName=None,
dedupByDefault=True,
drilldownFieldnamesTranslate=lambda s: s):
Observable.__init__(self)
self._resultsFrom = resultsFrom
self._matches = matches or []
self._cores = set(cSpec['core'] for match in self._matches
for cSpec in match)
self._dedupFieldName = dedupFieldName
self._dedupSortFieldName = dedupSortFieldName
self._dedupByDefault = dedupByDefault
self._drilldownFieldnamesTranslate = drilldownFieldnamesTranslate
self._clusteringEnabled = True
self._extraFilterQueries = ExtractFilterQueries(self._cores)
def __init__(self, resultsFrom, matches=None, dedupFieldName=None, dedupSortFieldName=None, dedupByDefault=True, drilldownFieldnamesTranslate=lambda s: s):
Observable.__init__(self)
self._resultsFrom = resultsFrom
self._matches = matches or []
self._cores = set(cSpec['core'] for match in self._matches for cSpec in match)
self._dedupFieldName = dedupFieldName
self._dedupSortFieldName = dedupSortFieldName
self._dedupByDefault = dedupByDefault
self._drilldownFieldnamesTranslate = drilldownFieldnamesTranslate
self._clusteringEnabled = True
self._extraFilterQueries = ExtractFilterQueries(self._cores)
class ConvertToComposedQuery(Observable):
def __init__(self, resultsFrom, matches=None, dedupFieldName=None, dedupSortFieldName=None, dedupByDefault=True, drilldownFieldnamesTranslate=lambda s: s):
Observable.__init__(self)
self._resultsFrom = resultsFrom
self._matches = matches or []
self._cores = set(cSpec['core'] for match in self._matches for cSpec in match)
self._dedupFieldName = dedupFieldName
self._dedupSortFieldName = dedupSortFieldName
self._dedupByDefault = dedupByDefault
self._drilldownFieldnamesTranslate = drilldownFieldnamesTranslate
self._clusteringEnabled = True
self._extraFilterQueries = ExtractFilterQueries(self._cores)
@asyncnoreturnvalue
def updateConfig(self, config, indexConfig=None, **kwargs):
self._clusteringEnabled = 'clustering' not in config.get('features_disabled', [])
def executeQuery(self, query=None, extraArguments=None, facets=None, drilldownQueries=None, filterQueries=None, sortKeys=None, **kwargs):
if 'cqlAbstractSyntaxTree' in kwargs:
query = kwargs.pop('cqlAbstractSyntaxTree')
query = cqlToExpression(query)
extraArguments = extraArguments or {}
cq = ComposedQuery(self._resultsFrom)
for matchTuple in self._matches:
cq.addMatch(*matchTuple)
for key in ['start', 'stop', 'suggestionRequest']:
if key in kwargs:
setattr(cq, key, kwargs[key])
coreQuery, filters = self._extraFilterQueries.convert(query, self._resultsFrom)
if coreQuery:
cq.setCoreQuery(core=self._resultsFrom, query=coreQuery)
for core, aFilter in ((core, aFilter) for core, filters in filters.items() for aFilter in filters):
cq.addFilterQuery(core, aFilter)
for sortKey in sortKeys or []:
core, sortBy = self._parseCorePrefix(sortKey['sortBy'], self._cores)
cq.addSortKey(dict(sortKey, core=core, sortBy=sortBy))
filters = extraArguments.get('x-filter', [])
for f in filters:
core, filterQuery = self._coreQuery(query=f, cores=self._cores)
cq.addFilterQuery(core=core, query=filterQuery)
for core, filterQuery in filterQueries or []:
cq.addFilterQuery(core=core, query=cqlToExpression(filterQuery))
rankQueries = extraArguments.get('x-rank-query', [])
if rankQueries:
queries = defaultdict(list)
for rankQuery in rankQueries:
core, rankQuery = self._parseCorePrefix(rankQuery, self._cores)
queries[core].append(rankQuery)
for core, q in queries.items():
cq.setRankQuery(core=core, query=cqlToExpression(' OR '.join(q)))
if self._dedupFieldName:
if 'true' == extraArguments.get('x-filter-common-keys', ['true' if self._dedupByDefault else 'false'])[0]:
setattr(cq, "dedupField", self._dedupFieldName)
setattr(cq, "dedupSortField", self._dedupSortFieldName)
if self._clusteringEnabled and 'true' == extraArguments.get('x-clustering', [None])[0]:
setattr(cq, "clustering", True)
facetOrder = []
fieldTranslations = {}
for drilldownField in (facets or []):
path = drilldownField['fieldname'].split('>')
fieldname, path = path[0], path[1:]
facetOrder.append((fieldname, path))
core, newFieldname = self._coreFacet(fieldname, self._cores)
newFieldname = self._drilldownFieldnamesTranslate(newFieldname)
fieldTranslations[newFieldname] = fieldname
cq.addFacet(core=core, facet=dict(fieldname=newFieldname, path=path, maxTerms=drilldownField['maxTerms']))
for drilldownQuery in (drilldownQueries or []):
core, fieldname = self._coreDrilldownQuery(drilldownQuery[0], self._cores)
fieldname = self._drilldownFieldnamesTranslate(fieldname)
cq.addDrilldownQuery(core=core, drilldownQuery=(fieldname, drilldownQuery[1]))
result = yield self.any.executeComposedQuery(query=cq)
drilldownData = getattr(result, "drilldownData", None)
if drilldownData:
for facet in drilldownData:
fieldname = facet['fieldname']
facet['fieldname'] = fieldTranslations.get(fieldname, fieldname)
result.drilldownData = sorted(
drilldownData,
key=lambda d: facetOrder.index((d['fieldname'], d.get('path', [])))
)
raise StopIteration(result)
def _coreQuery(self, query, cores):
core, query = self._parseCorePrefix(query, cores)
return core, cqlToExpression(query)
def _coreFacet(self, fieldname, cores):
return self._parseCorePrefix(fieldname, cores)
def _coreDrilldownQuery(self, drilldownQueryField, cores):
return self._parseCorePrefix(drilldownQueryField, cores)
def _parseCorePrefix(self, field, cores):
if field.startswith(self._resultsFrom):
return self._resultsFrom, field
core = self._resultsFrom
try:
tmpcore, tail = field.split('.', 1)
if tmpcore in cores:
core = tmpcore
field = tail
except ValueError:
pass
return core, field
def setUp(self):
super(ExtractFilterQueriesTest, self).setUp()
self.extractFilterQueries = ExtractFilterQueries(['core1', 'core2'])
self.convert = self.extractFilterQueries.convert
class ConvertToComposedQuery(Observable):
def __init__(self,
resultsFrom,
matches=None,
dedupFieldName=None,
dedupSortFieldName=None,
dedupByDefault=True,
drilldownFieldnamesTranslate=lambda s: s):
Observable.__init__(self)
self._resultsFrom = resultsFrom
self._matches = matches or []
self._cores = set(cSpec['core'] for match in self._matches
for cSpec in match)
self._dedupFieldName = dedupFieldName
self._dedupSortFieldName = dedupSortFieldName
self._dedupByDefault = dedupByDefault
self._drilldownFieldnamesTranslate = drilldownFieldnamesTranslate
self._clusteringEnabled = True
self._extraFilterQueries = ExtractFilterQueries(self._cores)
@asyncnoreturnvalue
def updateConfig(self, config, indexConfig=None, **kwargs):
self._clusteringEnabled = 'clustering' not in config.get(
'features_disabled', [])
def executeQuery(self,
query=None,
extraArguments=None,
facets=None,
drilldownQueries=None,
filterQueries=None,
sortKeys=None,
**kwargs):
if 'cqlAbstractSyntaxTree' in kwargs:
query = kwargs.pop('cqlAbstractSyntaxTree')
query = cqlToExpression(query)
extraArguments = extraArguments or {}
cq = ComposedQuery(self._resultsFrom)
for matchTuple in self._matches:
cq.addMatch(*matchTuple)
for key in ['start', 'stop', 'suggestionRequest']:
if key in kwargs:
setattr(cq, key, kwargs[key])
coreQuery, filters = self._extraFilterQueries.convert(
query, self._resultsFrom)
if coreQuery:
cq.setCoreQuery(core=self._resultsFrom, query=coreQuery)
for core, aFilter in ((core, aFilter)
for core, filters in filters.items()
for aFilter in filters):
cq.addFilterQuery(core, aFilter)
for sortKey in sortKeys or []:
core, sortBy = self._parseCorePrefix(sortKey['sortBy'],
self._cores)
cq.addSortKey(dict(sortKey, core=core, sortBy=sortBy))
filters = extraArguments.get('x-filter', [])
for f in filters:
core, filterQuery = self._coreQuery(query=f, cores=self._cores)
cq.addFilterQuery(core=core, query=filterQuery)
for core, filterQuery in filterQueries or []:
cq.addFilterQuery(core=core, query=cqlToExpression(filterQuery))
rankQueries = extraArguments.get('x-rank-query', [])
if rankQueries:
queries = defaultdict(list)
for rankQuery in rankQueries:
core, rankQuery = self._parseCorePrefix(rankQuery, self._cores)
queries[core].append(rankQuery)
for core, q in queries.items():
cq.setRankQuery(core=core,
query=cqlToExpression(' OR '.join(q)))
filterCommonKeysField = extraArguments.get(
'x-filter-common-keys-field', [self._dedupFieldName])[0]
if filterCommonKeysField and 'true' == extraArguments.get(
'x-filter-common-keys',
['true' if self._dedupByDefault else 'false'])[0]:
cq.dedupField = ('' if filterCommonKeysField.startswith(KEY_PREFIX)
else KEY_PREFIX) + filterCommonKeysField
cq.dedupSortField = self._dedupSortFieldName
if self._clusteringEnabled and 'true' == extraArguments.get(
'x-clustering', [None])[0]:
setattr(cq, "clustering", True)
facetOrder = []
fieldTranslations = {}
for drilldownField in (facets or []):
path = drilldownField['fieldname'].split('>')
fieldname, path = path[0], path[1:]
facetOrder.append((fieldname, path))
core, newFieldname = self._coreFacet(fieldname, self._cores)
newFieldname = self._drilldownFieldnamesTranslate(newFieldname)
fieldTranslations[newFieldname] = fieldname
cq.addFacet(core=core,
facet=dict(fieldname=newFieldname,
path=path,
maxTerms=drilldownField['maxTerms']))
for drilldownQuery in (drilldownQueries or []):
core, fieldname = self._coreDrilldownQuery(drilldownQuery[0],
self._cores)
fieldname = self._drilldownFieldnamesTranslate(fieldname)
cq.addDrilldownQuery(core=core,
drilldownQuery=(fieldname, drilldownQuery[1]))
result = yield self.any.executeComposedQuery(query=cq)
drilldownData = getattr(result, "drilldownData", None)
if drilldownData:
for facet in drilldownData:
fieldname = facet['fieldname']
facet['fieldname'] = fieldTranslations.get(
fieldname, fieldname)
result.drilldownData = sorted(
drilldownData,
key=lambda d: facetOrder.index(
(d['fieldname'], d.get('path', []))))
raise StopIteration(result)
def _coreQuery(self, query, cores):
core, query = self._parseCorePrefix(query, cores)
return core, cqlToExpression(query)
def _coreFacet(self, fieldname, cores):
return self._parseCorePrefix(fieldname, cores)
def _coreDrilldownQuery(self, drilldownQueryField, cores):
return self._parseCorePrefix(drilldownQueryField, cores)
def _parseCorePrefix(self, field, cores):
if field.startswith(self._resultsFrom):
return self._resultsFrom, field
core = self._resultsFrom
try:
tmpcore, tail = field.split('.', 1)
if tmpcore in cores:
core = tmpcore
field = tail
except ValueError:
pass
return core, field
