def cleanup(self, batch=1000): """ remove entries from solr that don't have a corresponding Zope object or have a different UID than the real object""" manager = queryUtility(ISolrConnectionManager) proc = SolrIndexProcessor(manager) conn = manager.getConnection() log = self.mklog(use_std_log=True) log('cleaning up solr index...\n') key = manager.getSchema().uniqueKey start = 0 resp = SolrResponse(conn.search(q='*:*', rows=batch, start=start)) res = resp.results() log('%s items in solr catalog\n' % resp.response.numFound) deleted = 0 reindexed = 0 while len(res) > 0: for flare in res: try: ob = PloneFlare(flare).getObject() except Exception as err: log('Error getting object, removing: %s (%s)\n' % ( flare['path_string'], err)) conn.delete(flare[key]) deleted += 1 continue if not IUUIDAware.providedBy(ob): log('Object %s of type %s does not support uuids, skipping.\n' % ('/'.join(ob.getPhysicalPath()), ob.meta_type)) continue uuid = IUUID(ob) if uuid != flare[key]: log('indexed under wrong UID, removing: %s\n' % flare['path_string']) conn.delete(flare[key]) deleted += 1 realob_res = SolrResponse(conn.search(q='%s:%s' % (key, uuid))).results() if len(realob_res) == 0: log('no sane entry for last object, reindexing\n') data, missing = proc.getData(ob) prepareData(data) if not missing: boost = boost_values(ob, data) conn.add(boost_values=boost, **data) reindexed += 1 else: log(' missing data, cannot index.\n') log('handled batch of %d items, commiting\n' % len(res)) conn.commit() start += batch resp = SolrResponse(conn.search(q='*:*', rows=batch, start=start)) res = resp.results() msg = 'solr cleanup finished, %s item(s) removed, %s item(s) reindexed\n' % (deleted, reindexed) log(msg) logger.info(msg)
def test_breadcrumbs_cropping(self): portal = self.layer['portal'] request = self.layer['request'] # Setup browser layers notify(BeforeTraverseEvent(portal, request)) # Enable path based breadcrumbs registry = getUtility(IRegistry) settings = registry.forInterface(ISearchSettings) settings.path_based_breadcrumbs = True flare = PloneFlare(portal) flare['path_string'] = '/plone/a/long/path/to/an/object' view = getMultiAdapter((portal, request), name=u'search') breadcrumbs = view.breadcrumbs(flare) self.assertEquals(len(breadcrumbs), 4) self.assertEquals(breadcrumbs[0]['Title'], 'a') self.assertEquals(breadcrumbs[1]['Title'], u'…') self.assertEquals(breadcrumbs[2]['Title'], 'to') self.assertEquals(breadcrumbs[3]['Title'], 'an') settings.max_breadcrumbs = 5 breadcrumbs = view.breadcrumbs(flare) self.assertEquals(len(breadcrumbs), 5) self.assertEquals(breadcrumbs[0]['Title'], 'a') self.assertEquals(breadcrumbs[1]['Title'], 'long') self.assertEquals(breadcrumbs[2]['Title'], 'path') self.assertEquals(breadcrumbs[3]['Title'], 'to') self.assertEquals(breadcrumbs[4]['Title'], 'an')
def search_solr(query, request=None, lang_query=True, **params): search = queryUtility(ISearch) if lang_query: dummy = {} languageFilter(dummy) prepareData(dummy) # this replaces '' with 'any' langquery = 'Language:(%s)' % ' OR '.join(dummy['Language']) query = '(%s) AND %s' % (query, langquery) response = search(query, **params) if request is None: request = getSite().REQUEST response.request = request results = response.results() for idx, flare in enumerate(results): results[idx] = PloneFlare(flare, request=request) padResults(results, **params) # pad the batch return response
def setUp(self): self.flare = FlareContentListingObject(PloneFlare({ 'getId': 'foobar', 'path_string': '/plone/news', 'UID': 'test-uid', 'getObjSize': 42, 'review_state': 'published', 'listCreators': ['foo', 'bar'], 'Creator': 'Flare Creator', 'Title': 'Flare Title', 'Description': 'Flare Description', 'Subject': 'Flare Subject', 'Date': 'Flare Date', 'expires': '1.1.2099', 'created': '31.12.1969', 'modified': '27.07.2016', 'Language': 'de', 'portal_type': 'NewsItem', 'Type': 'Flare NewsItem', }))
def setUp(self): self.flare = FlareContentListingObject( PloneFlare({ "getId": "foobar", "path_string": "/plone/news", "UID": "test-uid", "getObjSize": 42, "review_state": "published", "listCreators": ["foo", "bar"], "Creator": "Flare Creator", "Title": "Flare Title", "Description": "Flare Description", "Subject": "Flare Subject", "Date": "Flare Date", "expires": DateTime("1.1.2099"), "created": DateTime("31.12.1969"), "modified": DateTime("27.07.2016"), "Language": "de", "portal_type": "NewsItem", "Type": "Flare NewsItem", }))
def test_breadcrumbs(self): portal = self.layer['portal'] request = self.layer['request'] # Setup browser layers notify(BeforeTraverseEvent(portal, request)) setRoles(portal, TEST_USER_ID, ['Manager']) f1 = portal[portal.invokeFactory('Folder', 'f1', title=u"Folder 1")] f2 = f1[f1.invokeFactory('Folder', 'f2', title=u"Folder 2")] f3 = f2[f2.invokeFactory('Folder', 'f3', title=u"Folder 3")] flare = PloneFlare(portal) flare['path_string'] = '/plone/f1/f2/f3' view = getMultiAdapter((f3, request), name=u'search') breadcrumbs = view.breadcrumbs(flare) self.assertEquals(len(breadcrumbs), 2) self.assertEquals(breadcrumbs[0]['Title'], 'Folder 1') self.assertEquals(breadcrumbs[0]['absolute_url'], f1.absolute_url()) self.assertEquals(breadcrumbs[1]['Title'], 'Folder 2') self.assertEquals(breadcrumbs[1]['absolute_url'], f2.absolute_url())
def test_path_based_breadcrumbs(self): portal = self.layer['portal'] request = self.layer['request'] # Setup browser layers notify(BeforeTraverseEvent(portal, request)) # Enable path based breadcrumbs registry = getUtility(IRegistry) settings = registry.forInterface(ISearchSettings) settings.path_based_breadcrumbs = True flare = PloneFlare(portal) flare['path_string'] = '/plone/path/to/object' view = getMultiAdapter((portal, request), name=u'search') breadcrumbs = view.breadcrumbs(flare) self.assertEquals(len(breadcrumbs), 2) self.assertEquals(breadcrumbs[0]['Title'], 'path') self.assertEquals(breadcrumbs[0]['absolute_url'], portal.absolute_url() + '/path') self.assertEquals(breadcrumbs[1]['Title'], 'to') self.assertEquals(breadcrumbs[1]['absolute_url'], portal.absolute_url() + '/path/to')
def search_solr(self, query, **parameters): lang_tool = api.portal.get_tool("portal_languages") search = queryUtility(ISearch) if search is None: log.warn('Could not get solr ISearch utility') return [] rc = api.portal.get_tool("reference_catalog") preferred_lang = lang_tool.getPreferredLanguage() languages = ["en", "any"] # Speed approach # * Search for all en and neutral items with a given batch size # * If current language is en: return # * Take the found items and feed their UIDs into the reference catalog # * Look up any items that are translations of these in the current # language # * Replace the items with their translations # * Pad the result # * Return result query = ' '.join( (query, "+Language:({0})".format(' OR '.join(languages)))) if 'rows' not in parameters: parameters['rows'] = 100000 if 'start' not in parameters: parameters['start'] = 0 solr_response = search(query, **parameters) if not solr_response: return solr_response results = solr_response.results() schema = search.getManager().getSchema() or {} #found_results = results[ # parameters[start]:parameters[start]+parameters[rows]] original_uids = [x.UID for x in results] translations = rc.search({ "relationship": "translationOf", "targetUID": original_uids, }) # maps canonicals to translations target_map = {} # maps translations to canonicals source_map = {} # used to store translations under the uid of the canonical translation_map = {} # create a map where every translation is stored under the UID # of the canonical for t in translations: if t.Language != preferred_lang: continue target_map[t['targetUID']] = t['sourceUID'] source_map[t['sourceUID']] = t['targetUID'] # search for the full brains of the translations if target_map: t_query = "Language:%s AND UID: (%s)" % ( preferred_lang, ' OR '.join(target_map.values())) translation_response = search(t_query) for item in translation_response: targetUID = source_map[item.UID] if item.Language == preferred_lang: translation_map[targetUID] = item for idx, flare in enumerate(results): if flare.UID not in translation_map.keys(): flare = PloneFlare(flare) else: flare = PloneFlare(translation_map[flare.UID]) for missing in set(schema.stored).difference(flare): flare[missing] = MV results[idx] = flare padResults(results, **parameters) return solr_response
def cleanup(self, batch=1000): """remove entries from solr that don't have a corresponding Zope object or have a different UID than the real object""" manager = queryUtility(ISolrConnectionManager) proc = SolrIndexProcessor(manager) conn = manager.getConnection() log = self.mklog(use_std_log=True) log("cleaning up solr index...\n") key = manager.getSchema().uniqueKey start = 0 resp = SolrResponse(conn.search(q="*:*", rows=batch, start=start)) res = resp.results() log("%s items in solr catalog\n" % resp.response.numFound) deleted = 0 reindexed = 0 while len(res) > 0: for flare in res: try: ob = PloneFlare(flare).getObject() except Exception as err: log("Error getting object, removing: %s (%s)\n" % (flare["path_string"], err)) conn.delete(flare[key]) deleted += 1 continue if ob is None: log("Object not found, removing: %s\n" % (flare["path_string"])) conn.delete(flare[key]) deleted += 1 continue if not IUUIDAware.providedBy(ob): no_skipping_msg = ("Object %s of type %s does not " + "support uuids, skipping.\n") log(no_skipping_msg % ("/".join(ob.getPhysicalPath()), ob.meta_type)) continue uuid = IUUID(ob) if uuid != flare[key]: log("indexed under wrong UID, removing: %s\n" % flare["path_string"]) conn.delete(flare[key]) deleted += 1 realob_res = SolrResponse( conn.search(q="%s:%s" % (key, uuid))).results() if len(realob_res) == 0: log("no sane entry for last object, reindexing\n") data, missing = proc.getData(ob) prepareData(data) if not missing: boost = boost_values(ob, data) conn.add(boost_values=boost, **data) reindexed += 1 else: log(" missing data, cannot index.\n") log("handled batch of %d items, committing\n" % len(res)) conn.commit() start += batch resp = SolrResponse(conn.search(q="*:*", rows=batch, start=start)) res = resp.results() finished_msg = ("solr cleanup finished, %s item(s) removed, " + "%s item(s) reindexed\n") msg = finished_msg % (deleted, reindexed) log(msg) logger.info(msg)
def score(**kw): return PloneFlare(SolrFlare(**kw)).data_record_normalized_score_