def testResultsPaddingWithStart(self):
results = self.results()
padResults(results, start=50)
self.assertEqual(len(results), 1204)
self.assertEqual(results[:50], [None] * 50)
self.assertEqual(results[50].UID, '7c31adb20d5eee314233abfe48515cf3')
self.assertEqual(results[187:], [None] * (1204 - 187))
def testResultsPaddingWithStart(self):
results = self.results()
padResults(results, start=50)
self.assertEqual(len(results), 1204)
self.assertEqual(results[:50], [None] * 50)
self.assertEqual(results[50].UID, '7c31adb20d5eee314233abfe48515cf3')
self.assertEqual(results[187:], [None] * (1204 - 187))
def solrSearchResults(request=None, **keywords):
""" perform a query using solr after translating the passed in
parameters with portal catalog semantics """
site = getSite()
search = queryUtility(ISearch, context=site)
config = queryUtility(ISolrConnectionConfig, context=site)
if request is None:
# try to get a request instance, so that flares can be adapted to
# ploneflares and urls can be converted into absolute ones etc;
# however, in this case any arguments from the request are ignored
args = deepcopy(keywords)
request = getattr(site, 'REQUEST', None)
elif IHTTPRequest.providedBy(request):
args = deepcopy(request.form)
args.update(keywords) # keywords take precedence
else:
assert isinstance(request, dict), request
args = deepcopy(request)
args.update(keywords) # keywords take precedence
# if request is a dict, we need the real request in order to
# be able to adapt to plone flares
request = getattr(site, 'REQUEST', args)
if 'path' in args and 'navtree' in args['path']:
raise FallBackException # we can't handle navtree queries yet
use_solr = args.get('use_solr', False) # A special key to force Solr
if not use_solr and config.required:
required = set(config.required).intersection(args)
if required:
for key in required:
if not args[key]:
raise FallBackException
else:
raise FallBackException
query, params = search.buildQueryAndParameters(**args)
if query != {}:
__traceback_info__ = (query, params, args)
response = search(query, **params)
else:
return SolrResponse()
def wrap(flare):
""" wrap a flare object with a helper class """
adapter = queryMultiAdapter((flare, request), IFlare)
return adapter is not None and adapter or flare
schema = search.getManager().getSchema() or {}
results = response.results()
for idx, flare in enumerate(results):
flare = wrap(flare)
for missing in set(schema.stored).difference(flare):
flare[missing] = MV
results[idx] = wrap(flare)
padResults(results, **params) # pad the batch
return response
def solrSearchResults(request=None, **keywords):
""" perform a query using solr after translating the passed in
parameters with portal catalog semantics """
site = getSite()
search = queryUtility(ISearch, context=site)
config = queryUtility(ISolrConnectionConfig, context=site)
if request is None:
# try to get a request instance, so that flares can be adapted to
# ploneflares and urls can be converted into absolute ones etc;
# however, in this case any arguments from the request are ignored
args = deepcopy(keywords)
request = getattr(site, 'REQUEST', None)
elif IHTTPRequest.providedBy(request):
args = deepcopy(request.form)
args.update(keywords) # keywords take precedence
else:
assert isinstance(request, dict), request
args = deepcopy(request)
args.update(keywords) # keywords take precedence
# if request is a dict, we need the real request in order to
# be able to adapt to plone flares
request = getattr(site, 'REQUEST', args)
if 'path' in args and 'navtree' in args['path']:
raise FallBackException # we can't handle navtree queries yet
use_solr = args.get('use_solr', False) # A special key to force Solr
if not use_solr and config.required:
required = set(config.required).intersection(args)
if required:
for key in required:
if not args[key]:
raise FallBackException
else:
raise FallBackException
query, params = search.buildQueryAndParameters(**args)
if query != {}:
__traceback_info__ = (query, params, args)
response = search(query, **params)
else:
return SolrResponse()
def wrap(flare):
""" wrap a flare object with a helper class """
adapter = queryMultiAdapter((flare, request), IFlare)
return adapter is not None and adapter or flare
schema = search.getManager().getSchema() or {}
results = response.results()
for idx, flare in enumerate(results):
flare = wrap(flare)
for missing in set(schema.stored).difference(flare):
flare[missing] = MV
results[idx] = wrap(flare)
padResults(results, **params) # pad the batch
return response
def search_solr(query, request=None, lang_query=True, **params):
search = queryUtility(ISearch)
if lang_query:
dummy = {}
languageFilter(dummy)
prepareData(dummy) # this replaces '' with 'any'
langquery = 'Language:(%s)' % ' OR '.join(dummy['Language'])
query = '(%s) AND %s' % (query, langquery)
response = search(query, **params)
if request is None:
request = getSite().REQUEST
response.request = request
results = response.results()
for idx, flare in enumerate(results):
results[idx] = PloneFlare(flare, request=request)
padResults(results, **params) # pad the batch
return response
def search_solr(query, request=None, lang_query=True, **params):
search = queryUtility(ISearch)
if lang_query:
dummy = {}
languageFilter(dummy)
prepareData(dummy) # this replaces '' with 'any'
langquery = 'Language:(%s)' % ' OR '.join(dummy['Language'])
query = '(%s) AND %s' % (query, langquery)
response = search(query, **params)
if request is None:
request = getSite().REQUEST
response.request = request
results = response.results()
for idx, flare in enumerate(results):
results[idx] = PloneFlare(flare, request=request)
padResults(results, **params) # pad the batch
return response
def search_solr(self, query, **parameters):
lang_tool = api.portal.get_tool("portal_languages")
search = queryUtility(ISearch)
if search is None:
log.warn('Could not get solr ISearch utility')
return []
rc = api.portal.get_tool("reference_catalog")
preferred_lang = lang_tool.getPreferredLanguage()
languages = ["en", "any"]
# Speed approach
# * Search for all en and neutral items with a given batch size
# * If current language is en: return
# * Take the found items and feed their UIDs into the reference catalog
# * Look up any items that are translations of these in the current
# language
# * Replace the items with their translations
# * Pad the result
# * Return result
query = ' '.join(
(query, "+Language:({0})".format(' OR '.join(languages))))
if 'rows' not in parameters:
parameters['rows'] = 100000
if 'start' not in parameters:
parameters['start'] = 0
solr_response = search(query, **parameters)
if not solr_response:
return solr_response
results = solr_response.results()
schema = search.getManager().getSchema() or {}
#found_results = results[
# parameters[start]:parameters[start]+parameters[rows]]
original_uids = [x.UID for x in results]
translations = rc.search({
"relationship": "translationOf",
"targetUID": original_uids,
})
# maps canonicals to translations
target_map = {}
# maps translations to canonicals
source_map = {}
# used to store translations under the uid of the canonical
translation_map = {}
# create a map where every translation is stored under the UID
# of the canonical
for t in translations:
if t.Language != preferred_lang:
continue
target_map[t['targetUID']] = t['sourceUID']
source_map[t['sourceUID']] = t['targetUID']
# search for the full brains of the translations
if target_map:
t_query = "Language:%s AND UID: (%s)" % (
preferred_lang, ' OR '.join(target_map.values()))
translation_response = search(t_query)
for item in translation_response:
targetUID = source_map[item.UID]
if item.Language == preferred_lang:
translation_map[targetUID] = item
for idx, flare in enumerate(results):
if flare.UID not in translation_map.keys():
flare = PloneFlare(flare)
else:
flare = PloneFlare(translation_map[flare.UID])
for missing in set(schema.stored).difference(flare):
flare[missing] = MV
results[idx] = flare
padResults(results, **parameters)
return solr_response
def testResultsPadding(self):
results = self.results()
padResults(results)
self.assertEqual(len(results), 1204)
self.assertEqual(results[0].UID, '7c31adb20d5eee314233abfe48515cf3')
self.assertEqual(results[137:], [None] * (1204 - 137))
def testResultsPadding(self):
results = self.results()
padResults(results)
self.assertEqual(len(results), 1204)
self.assertEqual(results[0].UID, '7c31adb20d5eee314233abfe48515cf3')
self.assertEqual(results[137:], [None] * (1204 - 137))
def search_solr(self, query, **parameters):
lang_tool = api.portal.get_tool("portal_languages")
search = queryUtility(ISearch)
if search is None:
log.warn('Could not get solr ISearch utility')
return []
rc = api.portal.get_tool("reference_catalog")
preferred_lang = lang_tool.getPreferredLanguage()
languages = ["en", "any"]
# Speed approach
# * Search for all en and neutral items with a given batch size
# * If current language is en: return
# * Take the found items and feed their UIDs into the reference catalog
# * Look up any items that are translations of these in the current
# language
# * Replace the items with their translations
# * Pad the result
# * Return result
query = ' '.join((
query, "+Language:({0})".format(' OR '.join(languages))))
if 'rows' not in parameters:
parameters['rows'] = 100000
if 'start' not in parameters:
parameters['start'] = 0
solr_response = search(query, **parameters)
if not solr_response:
return solr_response
results = solr_response.results()
schema = search.getManager().getSchema() or {}
#found_results = results[
# parameters[start]:parameters[start]+parameters[rows]]
original_uids = [x.UID for x in results]
translations = rc.search({
"relationship": "translationOf",
"targetUID": original_uids,
})
# maps canonicals to translations
target_map = {}
# maps translations to canonicals
source_map = {}
# used to store translations under the uid of the canonical
translation_map = {}
# create a map where every translation is stored under the UID
# of the canonical
for t in translations:
if t.Language != preferred_lang:
continue
target_map[t['targetUID']] = t['sourceUID']
source_map[t['sourceUID']] = t['targetUID']
# search for the full brains of the translations
if target_map:
t_query = "Language:%s AND UID: (%s)" % (
preferred_lang, ' OR '.join(target_map.values()))
translation_response = search(t_query)
for item in translation_response:
targetUID = source_map[item.UID]
if item.Language == preferred_lang:
translation_map[targetUID] = item
for idx, flare in enumerate(results):
if flare.UID not in translation_map.keys():
flare = PloneFlare(flare)
else:
flare = PloneFlare(translation_map[flare.UID])
for missing in set(schema.stored).difference(flare):
flare[missing] = MV
results[idx] = flare
padResults(results, **parameters)
return solr_response
