def cleanup(self, batch=1000):
""" remove entries from solr that don't have a corresponding Zope
object or have a different UID than the real object"""
manager = queryUtility(ISolrConnectionManager)
proc = SolrIndexProcessor(manager)
conn = manager.getConnection()
log = self.mklog(use_std_log=True)
log('cleaning up solr index...\n')
key = manager.getSchema().uniqueKey
start = 0
resp = SolrResponse(conn.search(q='*:*', rows=batch, start=start))
res = resp.results()
log('%s items in solr catalog\n' % resp.response.numFound)
deleted = 0
reindexed = 0
while len(res) > 0:
for flare in res:
try:
ob = PloneFlare(flare).getObject()
except Exception as err:
log('Error getting object, removing: %s (%s)\n' % (
flare['path_string'], err))
conn.delete(flare[key])
deleted += 1
continue
if not IUUIDAware.providedBy(ob):
log('Object %s of type %s does not support uuids, skipping.\n' %
('/'.join(ob.getPhysicalPath()), ob.meta_type))
continue
uuid = IUUID(ob)
if uuid != flare[key]:
log('indexed under wrong UID, removing: %s\n' %
flare['path_string'])
conn.delete(flare[key])
deleted += 1
realob_res = SolrResponse(conn.search(q='%s:%s' %
(key, uuid))).results()
if len(realob_res) == 0:
log('no sane entry for last object, reindexing\n')
data, missing = proc.getData(ob)
prepareData(data)
if not missing:
boost = boost_values(ob, data)
conn.add(boost_values=boost, **data)
reindexed += 1
else:
log(' missing data, cannot index.\n')
log('handled batch of %d items, commiting\n' % len(res))
conn.commit()
start += batch
resp = SolrResponse(conn.search(q='*:*', rows=batch, start=start))
res = resp.results()
msg = 'solr cleanup finished, %s item(s) removed, %s item(s) reindexed\n' % (deleted, reindexed)
log(msg)
logger.info(msg)
def test_breadcrumbs_cropping(self):
portal = self.layer['portal']
request = self.layer['request']
# Setup browser layers
notify(BeforeTraverseEvent(portal, request))
# Enable path based breadcrumbs
registry = getUtility(IRegistry)
settings = registry.forInterface(ISearchSettings)
settings.path_based_breadcrumbs = True
flare = PloneFlare(portal)
flare['path_string'] = '/plone/a/long/path/to/an/object'
view = getMultiAdapter((portal, request), name=u'search')
breadcrumbs = view.breadcrumbs(flare)
self.assertEquals(len(breadcrumbs), 4)
self.assertEquals(breadcrumbs[0]['Title'], 'a')
self.assertEquals(breadcrumbs[1]['Title'], u'…')
self.assertEquals(breadcrumbs[2]['Title'], 'to')
self.assertEquals(breadcrumbs[3]['Title'], 'an')
settings.max_breadcrumbs = 5
breadcrumbs = view.breadcrumbs(flare)
self.assertEquals(len(breadcrumbs), 5)
self.assertEquals(breadcrumbs[0]['Title'], 'a')
self.assertEquals(breadcrumbs[1]['Title'], 'long')
self.assertEquals(breadcrumbs[2]['Title'], 'path')
self.assertEquals(breadcrumbs[3]['Title'], 'to')
self.assertEquals(breadcrumbs[4]['Title'], 'an')
def search_solr(query, request=None, lang_query=True, **params):
search = queryUtility(ISearch)
if lang_query:
dummy = {}
languageFilter(dummy)
prepareData(dummy) # this replaces '' with 'any'
langquery = 'Language:(%s)' % ' OR '.join(dummy['Language'])
query = '(%s) AND %s' % (query, langquery)
response = search(query, **params)
if request is None:
request = getSite().REQUEST
response.request = request
results = response.results()
for idx, flare in enumerate(results):
results[idx] = PloneFlare(flare, request=request)
padResults(results, **params) # pad the batch
return response
def setUp(self):
self.flare = FlareContentListingObject(PloneFlare({
'getId': 'foobar',
'path_string': '/plone/news',
'UID': 'test-uid',
'getObjSize': 42,
'review_state': 'published',
'listCreators': ['foo', 'bar'],
'Creator': 'Flare Creator',
'Title': 'Flare Title',
'Description': 'Flare Description',
'Subject': 'Flare Subject',
'Date': 'Flare Date',
'expires': '1.1.2099',
'created': '31.12.1969',
'modified': '27.07.2016',
'Language': 'de',
'portal_type': 'NewsItem',
'Type': 'Flare NewsItem',
}))
def setUp(self):
self.flare = FlareContentListingObject(
PloneFlare({
"getId": "foobar",
"path_string": "/plone/news",
"UID": "test-uid",
"getObjSize": 42,
"review_state": "published",
"listCreators": ["foo", "bar"],
"Creator": "Flare Creator",
"Title": "Flare Title",
"Description": "Flare Description",
"Subject": "Flare Subject",
"Date": "Flare Date",
"expires": DateTime("1.1.2099"),
"created": DateTime("31.12.1969"),
"modified": DateTime("27.07.2016"),
"Language": "de",
"portal_type": "NewsItem",
"Type": "Flare NewsItem",
}))
def test_breadcrumbs(self):
portal = self.layer['portal']
request = self.layer['request']
# Setup browser layers
notify(BeforeTraverseEvent(portal, request))
setRoles(portal, TEST_USER_ID, ['Manager'])
f1 = portal[portal.invokeFactory('Folder', 'f1', title=u"Folder 1")]
f2 = f1[f1.invokeFactory('Folder', 'f2', title=u"Folder 2")]
f3 = f2[f2.invokeFactory('Folder', 'f3', title=u"Folder 3")]
flare = PloneFlare(portal)
flare['path_string'] = '/plone/f1/f2/f3'
view = getMultiAdapter((f3, request), name=u'search')
breadcrumbs = view.breadcrumbs(flare)
self.assertEquals(len(breadcrumbs), 2)
self.assertEquals(breadcrumbs[0]['Title'], 'Folder 1')
self.assertEquals(breadcrumbs[0]['absolute_url'], f1.absolute_url())
self.assertEquals(breadcrumbs[1]['Title'], 'Folder 2')
self.assertEquals(breadcrumbs[1]['absolute_url'], f2.absolute_url())
def test_path_based_breadcrumbs(self):
portal = self.layer['portal']
request = self.layer['request']
# Setup browser layers
notify(BeforeTraverseEvent(portal, request))
# Enable path based breadcrumbs
registry = getUtility(IRegistry)
settings = registry.forInterface(ISearchSettings)
settings.path_based_breadcrumbs = True
flare = PloneFlare(portal)
flare['path_string'] = '/plone/path/to/object'
view = getMultiAdapter((portal, request), name=u'search')
breadcrumbs = view.breadcrumbs(flare)
self.assertEquals(len(breadcrumbs), 2)
self.assertEquals(breadcrumbs[0]['Title'], 'path')
self.assertEquals(breadcrumbs[0]['absolute_url'],
portal.absolute_url() + '/path')
self.assertEquals(breadcrumbs[1]['Title'], 'to')
self.assertEquals(breadcrumbs[1]['absolute_url'],
portal.absolute_url() + '/path/to')
def search_solr(self, query, **parameters):
lang_tool = api.portal.get_tool("portal_languages")
search = queryUtility(ISearch)
if search is None:
log.warn('Could not get solr ISearch utility')
return []
rc = api.portal.get_tool("reference_catalog")
preferred_lang = lang_tool.getPreferredLanguage()
languages = ["en", "any"]
# Speed approach
# * Search for all en and neutral items with a given batch size
# * If current language is en: return
# * Take the found items and feed their UIDs into the reference catalog
# * Look up any items that are translations of these in the current
# language
# * Replace the items with their translations
# * Pad the result
# * Return result
query = ' '.join(
(query, "+Language:({0})".format(' OR '.join(languages))))
if 'rows' not in parameters:
parameters['rows'] = 100000
if 'start' not in parameters:
parameters['start'] = 0
solr_response = search(query, **parameters)
if not solr_response:
return solr_response
results = solr_response.results()
schema = search.getManager().getSchema() or {}
#found_results = results[
# parameters[start]:parameters[start]+parameters[rows]]
original_uids = [x.UID for x in results]
translations = rc.search({
"relationship": "translationOf",
"targetUID": original_uids,
})
# maps canonicals to translations
target_map = {}
# maps translations to canonicals
source_map = {}
# used to store translations under the uid of the canonical
translation_map = {}
# create a map where every translation is stored under the UID
# of the canonical
for t in translations:
if t.Language != preferred_lang:
continue
target_map[t['targetUID']] = t['sourceUID']
source_map[t['sourceUID']] = t['targetUID']
# search for the full brains of the translations
if target_map:
t_query = "Language:%s AND UID: (%s)" % (
preferred_lang, ' OR '.join(target_map.values()))
translation_response = search(t_query)
for item in translation_response:
targetUID = source_map[item.UID]
if item.Language == preferred_lang:
translation_map[targetUID] = item
for idx, flare in enumerate(results):
if flare.UID not in translation_map.keys():
flare = PloneFlare(flare)
else:
flare = PloneFlare(translation_map[flare.UID])
for missing in set(schema.stored).difference(flare):
flare[missing] = MV
results[idx] = flare
padResults(results, **parameters)
return solr_response
def cleanup(self, batch=1000):
"""remove entries from solr that don't have a corresponding Zope
object or have a different UID than the real object"""
manager = queryUtility(ISolrConnectionManager)
proc = SolrIndexProcessor(manager)
conn = manager.getConnection()
log = self.mklog(use_std_log=True)
log("cleaning up solr index...\n")
key = manager.getSchema().uniqueKey
start = 0
resp = SolrResponse(conn.search(q="*:*", rows=batch, start=start))
res = resp.results()
log("%s items in solr catalog\n" % resp.response.numFound)
deleted = 0
reindexed = 0
while len(res) > 0:
for flare in res:
try:
ob = PloneFlare(flare).getObject()
except Exception as err:
log("Error getting object, removing: %s (%s)\n" %
(flare["path_string"], err))
conn.delete(flare[key])
deleted += 1
continue
if ob is None:
log("Object not found, removing: %s\n" %
(flare["path_string"]))
conn.delete(flare[key])
deleted += 1
continue
if not IUUIDAware.providedBy(ob):
no_skipping_msg = ("Object %s of type %s does not " +
"support uuids, skipping.\n")
log(no_skipping_msg %
("/".join(ob.getPhysicalPath()), ob.meta_type))
continue
uuid = IUUID(ob)
if uuid != flare[key]:
log("indexed under wrong UID, removing: %s\n" %
flare["path_string"])
conn.delete(flare[key])
deleted += 1
realob_res = SolrResponse(
conn.search(q="%s:%s" % (key, uuid))).results()
if len(realob_res) == 0:
log("no sane entry for last object, reindexing\n")
data, missing = proc.getData(ob)
prepareData(data)
if not missing:
boost = boost_values(ob, data)
conn.add(boost_values=boost, **data)
reindexed += 1
else:
log(" missing data, cannot index.\n")
log("handled batch of %d items, committing\n" % len(res))
conn.commit()
start += batch
resp = SolrResponse(conn.search(q="*:*", rows=batch, start=start))
res = resp.results()
finished_msg = ("solr cleanup finished, %s item(s) removed, " +
"%s item(s) reindexed\n")
msg = finished_msg % (deleted, reindexed)
log(msg)
logger.info(msg)
def score(**kw):
return PloneFlare(SolrFlare(**kw)).data_record_normalized_score_