def testResultsPaddingWithStart(self): results = self.results() padResults(results, start=50) self.assertEqual(len(results), 1204) self.assertEqual(results[:50], [None] * 50) self.assertEqual(results[50].UID, '7c31adb20d5eee314233abfe48515cf3') self.assertEqual(results[187:], [None] * (1204 - 187))
def solrSearchResults(request=None, **keywords): """ perform a query using solr after translating the passed in parameters with portal catalog semantics """ site = getSite() search = queryUtility(ISearch, context=site) config = queryUtility(ISolrConnectionConfig, context=site) if request is None: # try to get a request instance, so that flares can be adapted to # ploneflares and urls can be converted into absolute ones etc; # however, in this case any arguments from the request are ignored args = deepcopy(keywords) request = getattr(site, 'REQUEST', None) elif IHTTPRequest.providedBy(request): args = deepcopy(request.form) args.update(keywords) # keywords take precedence else: assert isinstance(request, dict), request args = deepcopy(request) args.update(keywords) # keywords take precedence # if request is a dict, we need the real request in order to # be able to adapt to plone flares request = getattr(site, 'REQUEST', args) if 'path' in args and 'navtree' in args['path']: raise FallBackException # we can't handle navtree queries yet use_solr = args.get('use_solr', False) # A special key to force Solr if not use_solr and config.required: required = set(config.required).intersection(args) if required: for key in required: if not args[key]: raise FallBackException else: raise FallBackException query, params = search.buildQueryAndParameters(**args) if query != {}: __traceback_info__ = (query, params, args) response = search(query, **params) else: return SolrResponse() def wrap(flare): """ wrap a flare object with a helper class """ adapter = queryMultiAdapter((flare, request), IFlare) return adapter is not None and adapter or flare schema = search.getManager().getSchema() or {} results = response.results() for idx, flare in enumerate(results): flare = wrap(flare) for missing in set(schema.stored).difference(flare): flare[missing] = MV results[idx] = wrap(flare) padResults(results, **params) # pad the batch return response
def search_solr(query, request=None, lang_query=True, **params): search = queryUtility(ISearch) if lang_query: dummy = {} languageFilter(dummy) prepareData(dummy) # this replaces '' with 'any' langquery = 'Language:(%s)' % ' OR '.join(dummy['Language']) query = '(%s) AND %s' % (query, langquery) response = search(query, **params) if request is None: request = getSite().REQUEST response.request = request results = response.results() for idx, flare in enumerate(results): results[idx] = PloneFlare(flare, request=request) padResults(results, **params) # pad the batch return response
def search_solr(self, query, **parameters): lang_tool = api.portal.get_tool("portal_languages") search = queryUtility(ISearch) if search is None: log.warn('Could not get solr ISearch utility') return [] rc = api.portal.get_tool("reference_catalog") preferred_lang = lang_tool.getPreferredLanguage() languages = ["en", "any"] # Speed approach # * Search for all en and neutral items with a given batch size # * If current language is en: return # * Take the found items and feed their UIDs into the reference catalog # * Look up any items that are translations of these in the current # language # * Replace the items with their translations # * Pad the result # * Return result query = ' '.join( (query, "+Language:({0})".format(' OR '.join(languages)))) if 'rows' not in parameters: parameters['rows'] = 100000 if 'start' not in parameters: parameters['start'] = 0 solr_response = search(query, **parameters) if not solr_response: return solr_response results = solr_response.results() schema = search.getManager().getSchema() or {} #found_results = results[ # parameters[start]:parameters[start]+parameters[rows]] original_uids = [x.UID for x in results] translations = rc.search({ "relationship": "translationOf", "targetUID": original_uids, }) # maps canonicals to translations target_map = {} # maps translations to canonicals source_map = {} # used to store translations under the uid of the canonical translation_map = {} # create a map where every translation is stored under the UID # of the canonical for t in translations: if t.Language != preferred_lang: continue target_map[t['targetUID']] = t['sourceUID'] source_map[t['sourceUID']] = t['targetUID'] # search for the full brains of the translations if target_map: t_query = "Language:%s AND UID: (%s)" % ( preferred_lang, ' OR '.join(target_map.values())) translation_response = search(t_query) for item in translation_response: targetUID = source_map[item.UID] if item.Language == preferred_lang: translation_map[targetUID] = item for idx, flare in enumerate(results): if flare.UID not in translation_map.keys(): flare = PloneFlare(flare) else: flare = PloneFlare(translation_map[flare.UID]) for missing in set(schema.stored).difference(flare): flare[missing] = MV results[idx] = flare padResults(results, **parameters) return solr_response
def testResultsPadding(self): results = self.results() padResults(results) self.assertEqual(len(results), 1204) self.assertEqual(results[0].UID, '7c31adb20d5eee314233abfe48515cf3') self.assertEqual(results[137:], [None] * (1204 - 137))
def search_solr(self, query, **parameters): lang_tool = api.portal.get_tool("portal_languages") search = queryUtility(ISearch) if search is None: log.warn('Could not get solr ISearch utility') return [] rc = api.portal.get_tool("reference_catalog") preferred_lang = lang_tool.getPreferredLanguage() languages = ["en", "any"] # Speed approach # * Search for all en and neutral items with a given batch size # * If current language is en: return # * Take the found items and feed their UIDs into the reference catalog # * Look up any items that are translations of these in the current # language # * Replace the items with their translations # * Pad the result # * Return result query = ' '.join(( query, "+Language:({0})".format(' OR '.join(languages)))) if 'rows' not in parameters: parameters['rows'] = 100000 if 'start' not in parameters: parameters['start'] = 0 solr_response = search(query, **parameters) if not solr_response: return solr_response results = solr_response.results() schema = search.getManager().getSchema() or {} #found_results = results[ # parameters[start]:parameters[start]+parameters[rows]] original_uids = [x.UID for x in results] translations = rc.search({ "relationship": "translationOf", "targetUID": original_uids, }) # maps canonicals to translations target_map = {} # maps translations to canonicals source_map = {} # used to store translations under the uid of the canonical translation_map = {} # create a map where every translation is stored under the UID # of the canonical for t in translations: if t.Language != preferred_lang: continue target_map[t['targetUID']] = t['sourceUID'] source_map[t['sourceUID']] = t['targetUID'] # search for the full brains of the translations if target_map: t_query = "Language:%s AND UID: (%s)" % ( preferred_lang, ' OR '.join(target_map.values())) translation_response = search(t_query) for item in translation_response: targetUID = source_map[item.UID] if item.Language == preferred_lang: translation_map[targetUID] = item for idx, flare in enumerate(results): if flare.UID not in translation_map.keys(): flare = PloneFlare(flare) else: flare = PloneFlare(translation_map[flare.UID]) for missing in set(schema.stored).difference(flare): flare[missing] = MV results[idx] = flare padResults(results, **parameters) return solr_response