def repl(cls, match):
groups = match.groupdict()
if groups.get('url'):
old_url = groups.get('url')
if not old_url.startswith('resolveuid/'):
site = getSite()
portal_url = site.portal_url
site_url = site.absolute_url()
if portal_url.isURLInPortal(old_url, cls.context or None):
path = old_url.replace('%s/' % site_url, '', 1).encode('utf-8')
suffix = []
content = None
while path:
content = site.unrestrictedTraverse(path, default=None)
if IUUIDAware.providedBy(content):
break
suffix.insert(0, path.split('/')[-1])
path = '/'.join(path.split('/')[:-1])
if content and IUUIDAware.providedBy(content):
uuid = IUUID(content)
suffix.insert(0, '')
new_url = 'resolveuid/%s' % uuid + '/'.join(suffix)
return match.string[match.start():match.end()].replace(old_url,
new_url)
return match.string[match.start():match.end()]
def repl(cls, match):
groups = match.groupdict()
if groups.get('url'):
old_url = groups.get('url')
if not old_url.startswith('resolveuid/'):
site = getSite()
portal_url = site.portal_url
site_url = site.absolute_url()
if portal_url.isURLInPortal(old_url, cls.context or None):
path = old_url.replace('%s/' % site_url, '', 1)
suffix = []
content = None
while path:
content = site.unrestrictedTraverse(path, default=None)
if IUUIDAware.providedBy(content):
break
suffix.insert(0, path.split('/')[-1])
path = '/'.join(path.split('/')[:-1])
if content and IUUIDAware.providedBy(content):
uuid = IUUID(content)
suffix.insert(0, '')
new_url = 'resolveuid/%s' % uuid + '/'.join(suffix)
return match.string[match.start():match.end()].replace(old_url,
new_url)
return match.string[match.start():match.end()]
def set_where(self, value):
if hasattr(value, 'REQUEST'):
self.data["where_uri"] = value.REQUEST.ACTUAL_URL
if IUUIDAware.providedBy(value):
self.data["where_uid"] = IUUID(value)
if hasattr(value, 'getPhysicalPath'):
self.data["where_path"] = '/'.join(value.getPhysicalPath())
def store_activity(document, event):
if not IUUIDAware.providedBy(document):
return
folder = document.aq_parent
utility = getUtility(IDigestUtility).store_activity(
folder, 'delete', title=document.title_or_id(), uid=IUUID(document))
def path2uid(context, link):
# unrestrictedTraverse requires a string on py3. see:
# https://github.com/zopefoundation/Zope/issues/674
if not link:
return ""
portal = getMultiAdapter(
(context, context.REQUEST), name="plone_portal_state"
).portal()
portal_url = portal.portal_url()
portal_path = "/".join(portal.getPhysicalPath())
path = link
context_url = context.absolute_url()
relative_up = len(context_url.split("/")) - len(portal_url.split("/"))
if path.startswith(portal_url):
path = path[len(portal_url) + 1 :]
if not path.startswith(portal_path):
path = "{portal_path}/{path}".format(
portal_path=portal_path, path=path.lstrip("/")
)
obj = portal.unrestrictedTraverse(path, None)
if obj is None or obj == portal:
return link
segments = path.split("/")
suffix = ""
while not IUUIDAware.providedBy(obj):
obj = aq_parent(obj)
suffix += "/" + segments.pop()
# check if obj is wrong because of acquisition
if "/".join(obj.getPhysicalPath()) != "/".join(segments):
return link
href = relative_up * "../" + "resolveuid/" + IUUID(obj)
if suffix:
href += suffix
return href
def get_current_id(self):
self.update()
if IUUIDAware.providedBy(self.context):
target = str(IUUID(self.context))
else:
return None
rid = self.normalize("r-%s-%s" % (self.memberid, target))
return rid
def store_activity(document, event):
if not IUUIDAware.providedBy(document):
return
folder = document.aq_parent
utility = getUtility(IDigestUtility).store_activity(folder,
'modify',
uid=IUUID(document))
def get_current_id(self):
self.update()
if IUUIDAware.providedBy(self.context):
target = str(IUUID(self.context))
else:
return None
rid = self.normalize("r-%s-%s" % (self.memberid, target))
return rid
def test_uuid(self):
doc = makeContent(self.folder,
portal_type='DDocument',
title='Foo')
self.assertTrue(IUUIDAware.providedBy(doc))
uuid = IUUID(doc, None)
self.assertTrue(uuid == doc.UID())
def test_uuid(self):
doc = makeContent(self.folder,
portal_type='DDocument',
title='Foo')
self.assertTrue(IUUIDAware.providedBy(doc))
uuid = IUUID(doc, None)
self.assertTrue(uuid == doc.UID())
def store_activity(document, event):
if not IUUIDAware.providedBy(document):
return
folder = document.aq_parent
utility = getUtility(IDigestUtility).store_activity(folder,
'delete',
title=document.title_or_id(),
uid=IUUID(document))
def cleanup(self, batch=1000):
""" remove entries from solr that don't have a corresponding Zope
object or have a different UID than the real object"""
manager = queryUtility(ISolrConnectionManager)
proc = SolrIndexProcessor(manager)
conn = manager.getConnection()
log = self.mklog(use_std_log=True)
log('cleaning up solr index...\n')
key = manager.getSchema().uniqueKey
start = 0
resp = SolrResponse(conn.search(q='*:*', rows=batch, start=start))
res = resp.results()
log('%s items in solr catalog\n' % resp.response.numFound)
deleted = 0
reindexed = 0
while len(res) > 0:
for flare in res:
try:
ob = PloneFlare(flare).getObject()
except Exception as err:
log('Error getting object, removing: %s (%s)\n' % (
flare['path_string'], err))
conn.delete(flare[key])
deleted += 1
continue
if not IUUIDAware.providedBy(ob):
log('Object %s of type %s does not support uuids, skipping.\n' %
('/'.join(ob.getPhysicalPath()), ob.meta_type))
continue
uuid = IUUID(ob)
if uuid != flare[key]:
log('indexed under wrong UID, removing: %s\n' %
flare['path_string'])
conn.delete(flare[key])
deleted += 1
realob_res = SolrResponse(conn.search(q='%s:%s' %
(key, uuid))).results()
if len(realob_res) == 0:
log('no sane entry for last object, reindexing\n')
data, missing = proc.getData(ob)
prepareData(data)
if not missing:
boost = boost_values(ob, data)
conn.add(boost_values=boost, **data)
reindexed += 1
else:
log(' missing data, cannot index.\n')
log('handled batch of %d items, commiting\n' % len(res))
conn.commit()
start += batch
resp = SolrResponse(conn.search(q='*:*', rows=batch, start=start))
res = resp.results()
msg = 'solr cleanup finished, %s item(s) removed, %s item(s) reindexed\n' % (deleted, reindexed)
log(msg)
logger.info(msg)
def store_activity(document, event):
if not IUUIDAware.providedBy(document):
return
if document.isTemporary():
return
folder = document.aq_parent
utility = getUtility(IDigestUtility).store_activity(folder,
'add',
uid=IUUID(document))
def store_activity(document, event):
if not IUUIDAware.providedBy(document):
return
if document.isTemporary():
return
folder = document.aq_parent
utility = getUtility(IDigestUtility).store_activity(folder,
'add',
uid=IUUID(document))
def store_activity(document, event):
if not ILayer.providedBy(getattr(document, 'REQUEST', None)):
return
if not IUUIDAware.providedBy(document):
return
folder = document.aq_parent
get_tool().store_activity(folder, 'delete',
title=document.title_or_id(),
uid=IUUID(document))
def get_useraction_info(self):
info = {}
if IUUIDAware.providedBy(self.context):
info["target"] = "%s" % IUUID(self.context)
else:
LOG.error("context is not IUUIDAware: %s" % self.context)
return
info["what"] = self.event
info["when"] = datetime.now()
info["who"] = self.mtool.getAuthenticatedMember().getId()
info["where"] = '/'.join(self.context.getPhysicalPath())
return info
def store_activity(document, event):
if IObjectInitializedEvent.providedBy(event):
return
elif IContainerModifiedEvent.providedBy(event):
return
elif not ILayer.providedBy(getattr(document, 'REQUEST', None)):
return
elif not IUUIDAware.providedBy(document):
return
folder = document.aq_parent
get_tool().store_activity(folder, 'modify', uid=IUUID(document))
def store_activity(document, event):
if not ILayer.providedBy(getattr(document, 'REQUEST', None)):
return
elif not IUUIDAware.providedBy(document):
return
elif document.isTemporary():
return
folder = aq_parent(document)
if not folder:
return
get_tool().store_activity(folder, 'add', uid=IUUID(document))
def store_activity(document, event):
if not ILayer.providedBy(getattr(document, 'REQUEST', None)):
return
elif not IUUIDAware.providedBy(document):
return
elif api.content.get_state(document) not in PUBLISHED_STATES:
return
folder = aq_parent(document)
if not folder:
return
get_tool().store_activity(folder, 'publish', uid=IUUID(document))
def path2uid(context, path):
# unrestrictedTraverse requires a string on py3. see:
# https://github.com/zopefoundation/Zope/issues/674
if not isinstance(path, str):
path = path.decode("utf-8")
obj = context.unrestrictedTraverse(path, None)
if obj is None:
return None, None
segments = path.split("/")
suffix = ""
while not IUUIDAware.providedBy(obj):
obj = aq_parent(obj)
suffix += "/" + segments.pop()
return IUUID(obj), suffix
def set_where(self, value):
if IPloneControlPanelForm.providedBy(value):
context = value.context
request = value.request
self.data["where_uri"] = request.ACTUAL_URL
if IUUIDAware.providedBy(context):
self.data["where_uid"] = IUUID(context)
elif hasattr(context, 'UID'):
self.data["where_uid"] = context.UID()
elif IPloneSiteRoot.providedBy(context):
self.data["where_uid"] = "Plone"
if hasattr(context, "getPhysicalPath"):
path = "/".join(value.context.getPhysicalPath())
self.data["where_path"] = path
else:
LOG.error("target is not a controlpanel form")
def string_uuid(obj):
""" Returns the uuid as string without hyphens. Takes either UUIDs, strings
with hyphens, objects which are IUUIDAware or objects which have a UID
attribute. (The latter works for catalog brains). """
if isinstance(obj, six.string_types):
obj = str(obj)
elif hasattr(obj, 'UID'):
if callable(obj.UID):
obj = obj.UID() # plone 4.3, because
else:
obj = obj.UID
elif IUUIDAware.providedBy(obj):
obj = IUUID(obj)
elif callable(obj):
obj = obj()
return UUID(str(obj)).hex
def string_uuid(obj):
""" Returns the uuid as string without hyphens. Takes either UUIDs, strings
with hyphens, objects which are IUUIDAware or objects which have a UID
attribute. (The latter works for catalog brains). """
if isinstance(obj, six.string_types):
obj = str(obj)
elif hasattr(obj, 'UID'):
if callable(obj.UID):
obj = obj.UID() # plone 4.3, because
else:
obj = obj.UID
elif IUUIDAware.providedBy(obj):
obj = IUUID(obj)
elif callable(obj):
obj = obj()
return UUID(str(obj)).hex
def my_fundraisers(self):
me = self.person()
if not me:
return []
pc = getToolByName(self.context, 'portal_catalog')
idvals = []
user_id = getattr(me, 'id', None)
if user_id:
idvals.append(user_id)
email = getattr(me, 'email', None)
if email:
idvals.append(email)
my_uuid = None
if IUUIDAware.providedBy(me):
my_uuid = IUUID(me)
if my_uuid:
idvals.append(my_uuid)
my_brains = pc(portal_type=self._fundraiser_type,
Creator=idvals, sort_order='reverse', sort_on='created')
return [b.getObject() for b in my_brains]
def string_uuid(obj):
""" Returns the uuid as string without hyphens. Takes either UUIDs, strings
with hyphens, objects which are IUUIDAware or objects which have a UID
attribute. (The latter works for catalog brains). """
from seantis.reservation.interfaces import IResource
if isinstance(obj, basestring):
obj = str(obj)
if IResource.providedBy(obj):
obj = obj.uuid()
elif hasattr(obj, 'UID'):
if callable(obj.UID):
obj = obj.UID() # plone 4.3, because
else:
obj = obj.UID
elif IUUIDAware.providedBy(obj):
obj = IUUID(obj)
elif callable(obj):
obj = obj()
return UUID(str(obj)).hex
def string_uuid(obj):
""" Returns the uuid as string without hyphens. Takes either UUIDs, strings
with hyphens, objects which are IUUIDAware or objects which have a UID
attribute. (The latter works for catalog brains). """
from seantis.reservation.interfaces import IResource
if isinstance(obj, basestring):
obj = str(obj)
if IResource.providedBy(obj):
obj = obj.uuid()
elif hasattr(obj, 'UID'):
if callable(obj.UID):
obj = obj.UID() # plone 4.3, because
else:
obj = obj.UID
elif IUUIDAware.providedBy(obj):
obj = IUUID(obj)
elif callable(obj):
obj = obj()
return UUID(str(obj)).hex
def my_fundraisers(self):
me = self.person()
if not me:
return []
pc = getToolByName(self.context, 'portal_catalog')
idvals = []
user_id = getattr(me, 'id', None)
if user_id:
idvals.append(user_id)
email = getattr(me, 'email', None)
if email:
idvals.append(email)
my_uuid = None
if IUUIDAware.providedBy(me):
my_uuid = IUUID(me)
if my_uuid:
idvals.append(my_uuid)
my_brains = pc(portal_type=self._fundraiser_type,
Creator=idvals,
sort_order='reverse',
sort_on='created')
return [b.getObject() for b in my_brains]
def store_activity(document, event):
if not event.oldParent:
# this is not a move, this is an adding
return
if not event.newParent:
# this is not a move, this is an adding
return
if not ILayer.providedBy(getattr(document, 'REQUEST', None)):
return
if not IUUIDAware.providedBy(document):
return
if event.oldParent == event.newParent:
# ignore renames
return
folder = aq_parent(document)
if not folder:
return
get_tool().store_activity(folder, 'move', uid=IUUID(document),
old_parent=IUUID(event.oldParent))
def cleanup(self, batch=1000):
"""remove entries from solr that don't have a corresponding Zope
object or have a different UID than the real object"""
manager = queryUtility(ISolrConnectionManager)
proc = SolrIndexProcessor(manager)
conn = manager.getConnection()
log = self.mklog(use_std_log=True)
log("cleaning up solr index...\n")
key = manager.getSchema().uniqueKey
start = 0
resp = SolrResponse(conn.search(q="*:*", rows=batch, start=start))
res = resp.results()
log("%s items in solr catalog\n" % resp.response.numFound)
deleted = 0
reindexed = 0
while len(res) > 0:
for flare in res:
try:
ob = PloneFlare(flare).getObject()
except Exception as err:
log("Error getting object, removing: %s (%s)\n" %
(flare["path_string"], err))
conn.delete(flare[key])
deleted += 1
continue
if ob is None:
log("Object not found, removing: %s\n" %
(flare["path_string"]))
conn.delete(flare[key])
deleted += 1
continue
if not IUUIDAware.providedBy(ob):
no_skipping_msg = ("Object %s of type %s does not " +
"support uuids, skipping.\n")
log(no_skipping_msg %
("/".join(ob.getPhysicalPath()), ob.meta_type))
continue
uuid = IUUID(ob)
if uuid != flare[key]:
log("indexed under wrong UID, removing: %s\n" %
flare["path_string"])
conn.delete(flare[key])
deleted += 1
realob_res = SolrResponse(
conn.search(q="%s:%s" % (key, uuid))).results()
if len(realob_res) == 0:
log("no sane entry for last object, reindexing\n")
data, missing = proc.getData(ob)
prepareData(data)
if not missing:
boost = boost_values(ob, data)
conn.add(boost_values=boost, **data)
reindexed += 1
else:
log(" missing data, cannot index.\n")
log("handled batch of %d items, committing\n" % len(res))
conn.commit()
start += batch
resp = SolrResponse(conn.search(q="*:*", rows=batch, start=start))
res = resp.results()
finished_msg = ("solr cleanup finished, %s item(s) removed, " +
"%s item(s) reindexed\n")
msg = finished_msg % (deleted, reindexed)
log(msg)
logger.info(msg)
