def delColumn(self, name, threshold=10000):
"""Deletes a row from the meta data schema"""
names = list(self.names)
_index = names.index(name)
threshold = threshold if threshold is not None else 10000
if name not in self.schema:
LOG.error('delColumn attempted to delete nonexistent '
'column %s.', str(name))
return
del names[_index]
# rebuild the schema
schema = {}
for i, name in enumerate(names):
schema[name] = i
self.schema = schema
self.names = tuple(names)
# update the brain
self.updateBrains()
# remove the column value from each record
if len(self):
_next_index = _index + 1
pghandler = ZLogHandler(threshold)
pghandler.init('Deleting %s column' % name, len(self))
for i, (key, value) in enumerate(self.data.iteritems()):
pghandler.report(i)
self.data[key] = value[:_index] + value[_next_index:]
pghandler.finish()
def update_getIcon(context):
typesToUpdate = {
'Location': ('link_icon.gif', 'link_icon.png'),
'Name': ('document_icon.gif', 'document_icon.png'),
'PleiadesVocabulary': ('folder_icon.gif', 'folder_icon.png'),
'PleiadesVocabularyTerm': ('document_icon.gif', 'document_icon.png'),
}
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon')
brains = search(portal_type=typesToUpdate.keys(), sort_on="path")
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
old_icon, new_icon = typesToUpdate[brain.portal_type]
if brain_icon != new_icon:
rid = brain.getRID()
record = metadata[rid]
new_record = list(record)
new_record[getIconPos] = new_icon
metadata[rid] = tuple(new_record)
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')
def uidDuplications(self):
pc = getToolByName(self, 'portal_catalog')
ptool = getToolByName(self, 'portal_url')
pghandler = ZLogHandler(1000)
portal = ptool.getPortalObject()
basecat = pc._catalog
index = basecat.indexes['UID']
size = len(index._index.keys())
pghandler.init('Remove stale items from catalog', size)
cnt = 0
for uid, rid in index._index.items():
path = basecat.paths[rid]
pghandler.report(cnt)
cnt += 1
try:
obj = portal.restrictedTraverse(path)
except:
# print "could not get obj for path %s, skipping" % path
# we ignore this for the moment
continue
parent = aq_parent(obj)
ids = [x['id'] for x in parent._objects]
if obj.getId() not in ids and obj.getId() not in parent.objectIds():
print "path %s not in parent " % path
del index._index[uid]
index._length.change(-1)
return "done"
def updateIconsInBrains(context, typesToUpdate=None):
"""Update getIcon metadata column in given types.
typesToUpdate must be a dictionary, for example: {
# portal_type: ('old_icon.gif', 'new_icon.png'),
'Document': ('document_icon.gif', 'document_icon.png'),
}
The portal_types must have an empty icon_expr, because that is the
main use case.
"""
if not typesToUpdate:
logger.warn('No typesToUpdate given for updateIconsInBrains.')
return
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon', None)
empty_icons = _types_with_empty_icons(context, typesToUpdate)
brains = search(portal_type=empty_icons, sort_on='path')
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
if not brain_icon:
continue
if getIconPos is not None:
_update_icon_in_single_brain(
brain, typesToUpdate, getIconPos, metadata)
else:
# If we don't have a standard catalog tool, fall back to the
# official API
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(
obj, brain_path, ['id'], True, pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')
def refresh_getIcon_catalog_metadata(context):
"""
get_icon redefined: now boolean:
needs to update metadata
true if item is type image or has image field (named 'image')
e.g. leadimage
see https://github.com/plone/Products.CMFPlone/issues/1226
"""
# Attention, this relies heavily on catalog internals.
# get the more visible zcatalog
zcatalog = getToolByName(context, 'portal_catalog')
# get the more hidden, inner (real) catalog implementation
catalog = zcatalog._catalog
try:
# Get the getIcon index value and
# check if there is a getIcon at all, this may not exist in some
# customizations, who knows, but always exists in default Plone
metadata_index = catalog.names.index('getIcon')
except ValueError:
logger.info('`getIcon` not in metadata, skip upgrade step')
return
cnt = 0
# search whole catalog
results = zcatalog.unrestrictedSearchResults()
num_objects = len(results)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
for brain in results:
pghandler.report(cnt)
# First get the new value
try:
obj = brain._unrestrictedGetObject()
except (AttributeError, KeyError, TypeError, NotFound):
continue
new_value = bool(getattr(aq_base(obj), 'image', False))
# We can now update the record with the new getIcon value
record = list(catalog.data[brain.getRID()])
record[metadata_index] = new_value
catalog.data[brain.getRID()] = tuple(record)
cnt += 1 # we are curious
# done
pghandler.finish()
logger.info('Reindexed `getIcon` for %s items', str(cnt))
def reindex_mime_type(context):
# Attention, this relies heavily on catalog internals.
# get the more visible zcatalog
zcatalog = getToolByName(context, 'portal_catalog')
# get the more hidden, inner (real) catalog implementation
catalog = zcatalog._catalog
try:
metadata_index = catalog.names.index('mime_type')
except ValueError:
logger.info('`mime_type` not in metadata, skip upgrade step')
return
# we are interested in dexterity and archtetype images and files:
ifaces = ['plone.app.contenttypes.interfaces.IImage',
'plone.app.contenttypes.interfaces.IFile',
'Products.ATContentTypes.interfaces.file.IFileContent',
'Products.ATContentTypes.interfaces.image.IImageContent']
cnt = 0
results = zcatalog.unrestrictedSearchResults(object_provides=ifaces)
num_objects = len(results)
pghandler = ZLogHandler(1000)
pghandler.init('Updating mime_type metadata', num_objects)
for brain in results:
pghandler.report(cnt)
try:
obj = brain._unrestrictedGetObject()
except (AttributeError, KeyError, TypeError, NotFound):
continue
if not obj:
continue
# First get the new value:
new_value = ''
try: # Dexterity
new_value = obj.content_type()
except TypeError: # Archetypes
new_value = obj.content_type
except AttributeError:
continue
# We can now update the record with the new mime_type value
record = list(catalog.data[brain.getRID()])
record[metadata_index] = new_value
catalog.data[brain.getRID()] = tuple(record)
cnt += 1
pghandler.finish()
logger.info('Reindexed `mime_type` for %s items', str(cnt))
def cleanup(self):
cat = getToolByName(self, "portal_catalog")
purl = getToolByName(self, "portal_url")
portal = purl.getPortalObject()
zcat = cat._catalog
pghandler = ZLogHandler(1000)
size = len(zcat.uids)
pghandler.init('Remove stale items from catalog', size)
cnt = removed = 0
for path in zcat.uids.keys():
pghandler.report(cnt)
cnt += 1
try:
obj = portal.restrictedTraverse(path)
except:
# XXX what to do?
log.warning("no object found for %s" % path)
cat.uncatalog_object(path)
removed += 1
log.info('Kill old path: %s' % path)
else:
if "/".join(obj.getPhysicalPath()) != path:
# saved under an old path! kill it
cat.uncatalog_object(path)
#index = zcat.uids[path]
#del zcat.uids[path]
#del zcat.paths[index]
#zcat._length.change(-1)
removed += 1
log.info('Kill old path: %s' % path)
else:
parent = aq_parent(obj)
ids = [x['id'] for x in parent._objects]
if obj.getId() not in ids and obj.getId() not in parent.objectIds():
cat.uncatalog_object(path)
removed += 1
log.info('ID not found as a child of current parent, kill it: %s' % path)
pghandler.finish()
log.info('Finished with the catalog, removed a total of %d items' % removed)
log.info("Length: %d, len(uids): %d, len(paths): %d" % (zcat._length.value,
len(zcat.uids), len(zcat.paths)))
transaction.commit()
def addColumn(self, name, default_value=None, threshold=10000):
"""Adds a row to the meta data schema"""
schema = self.schema
names = list(self.names)
threshold = threshold if threshold is not None else 10000
if name != name.strip():
# Someone could have mistakenly added a space at the end
# of the input field.
LOG.warning('stripped space from new column %r -> %r', name,
name.strip())
name = name.strip()
if name in schema:
raise CatalogError('The column %s already exists' % name)
if name[0] == '_':
raise CatalogError('Cannot cache fields beginning with "_"')
values = schema.values()
if values:
schema[name] = max(values) + 1
else:
schema[name] = 0
names.append(name)
if default_value in (None, ''):
default_value = MV
if len(self):
pghandler = ZLogHandler(threshold)
pghandler.init('Adding %s column' % name, len(self))
for i, (key, value) in enumerate(self.data.iteritems()):
pghandler.report(i)
self.data[key] = value + (default_value, )
pghandler.finish()
self.names = tuple(names)
self.schema = schema
# new column? update the brain
self.updateBrains()
def reindex_sortable_title(context):
from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
catalog = getToolByName(context, "portal_catalog")
_catalog = catalog._catalog
indexes = _catalog.indexes
sort_title_index = indexes.get("sortable_title", None)
if sort_title_index is None:
return
from Products.PluginIndexes.FieldIndex import FieldIndex
if not isinstance(sort_title_index, FieldIndex.FieldIndex):
return
change = []
pghandler = ZLogHandler(10000)
logger.info("Analyzing sortable_title index")
pghandler.init("Analyzing sortable_title index", len(sort_title_index))
for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
pghandler.report(i)
if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
if hasattr(rids, "keys"):
change.extend(list(rids.keys()))
else:
change.append(rids)
pghandler.finish()
update_metadata = "sortable_title" in _catalog.schema
pghandler = ZLogHandler(1000)
logger.info("Updating sortable_title index")
pghandler.init("Updating sortable_title index", len(change))
for i, rid in enumerate(change):
pghandler.report(i)
brain = _catalog[rid]
try:
obj = brain.getObject()
except (AttributeError, KeyError):
continue
if update_metadata:
obj.reindexObject()
else:
obj.reindexObject(idxs=["sortable_title"])
pghandler.finish()
logger.info("Updated `sortable_title` index.")
def reindex_sortable_title(context):
from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
catalog = getToolByName(context, 'portal_catalog')
_catalog = catalog._catalog
indexes = _catalog.indexes
sort_title_index = indexes.get('sortable_title', None)
if sort_title_index is None:
return
from Products.PluginIndexes.FieldIndex import FieldIndex
if not isinstance(sort_title_index, FieldIndex.FieldIndex):
return
change = []
pghandler = ZLogHandler(10000)
logger.info('Analyzing sortable_title index')
pghandler.init('Analyzing sortable_title index', len(sort_title_index))
for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
pghandler.report(i)
if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
if hasattr(rids, 'keys'):
change.extend(list(rids.keys()))
else:
change.append(rids)
pghandler.finish()
update_metadata = 'sortable_title' in _catalog.schema
pghandler = ZLogHandler(1000)
logger.info('Updating sortable_title index')
pghandler.init('Updating sortable_title index', len(change))
for i, rid in enumerate(change):
pghandler.report(i)
brain = _catalog[rid]
try:
obj = brain.getObject()
except (AttributeError, KeyError):
continue
if update_metadata:
obj.reindexObject()
else:
obj.reindexObject(idxs=['sortable_title'])
pghandler.finish()
logger.info('Updated `sortable_title` index.')
def agenda_item_project_set_summary_html(context):
"""Set the mimetype of the summary to text/html.
Otherwise when editing you get a bare textarea and cannot save it.
"""
catalog = getToolByName(context, 'portal_catalog')
brains = catalog(portal_type='AgendaItemProject')
num_objects = len(brains)
logger.info("Updating agenda item summaries.")
fixed = 0
pghandler = ZLogHandler(100)
pghandler.init('Setting mimetype html for agenda item summaries',
num_objects)
for index, brain in enumerate(brains):
pghandler.report(index)
obj = brain.getObject()
if obj.summary.mimetype != 'text/html':
obj.summary.mimetype = 'text/html'
fixed += 1
pghandler.finish()
logger.info("Have set summary mimetype to text/html for {0}/{1} agenda "
"items.".format(fixed, len(brains)))
def update_catalog_metadata(context, types=None):
"""Update catalog metadata.
Adapted from updateIconMetadata in plone.app.upgrade v40/betas.py.
"""
catalog = getToolByName(context, 'portal_catalog')
search = catalog.unrestrictedSearchResults
if types is None:
logger.info('Updating catalog metadata.')
brains = search(sort_on="path")
else:
if isinstance(types, basestring):
types = [types]
logger.info('Updating catalog metadata for %s.', ', '.join(types))
brains = search(portal_type=types, sort_on="path")
num_objects = len(brains)
# Yes, this logs quite often, but I just know I am going to thank
# myself for that... Feedback about progress is good for my heart.
# [Maurits]
pghandler = ZLogHandler(10)
pghandler.init('Updating catalog metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Done updating catalog metadata.')
def fix_themes_navigation(context):
""" Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
objects.
"""
ctool = getToolByName(context, 'portal_catalog')
brains = ctool.unrestrictedSearchResults(
object_provides='eea.themecentre.interfaces.IThemeCentre')
logger.info("Fixing eea.themecentre navigation ...")
pghandler = ZLogHandler(100)
pghandler.init('Fixing eea.themecentre navigation', len(brains))
for index, brain in enumerate(brains):
doc = brain.getObject()
if not INavigationRoot.providedBy(doc):
alsoProvides(doc, INavigationRoot)
pghandler.report(index)
if index % 100 == 0:
transaction.savepoint(optimistic=True)
transaction.savepoint(optimistic=True)
pghandler.finish()
logger.info("Fixing eea.themecentre navigation ... DONE")
def fix_themes_navigation(context):
""" Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
objects.
"""
ctool = getToolByName(context, 'portal_catalog')
brains = ctool.unrestrictedSearchResults(
object_provides='eea.themecentre.interfaces.IThemeCentre')
logger.info("Fixing eea.themecentre navigation ...")
pghandler = ZLogHandler(100)
pghandler.init('Fixing eea.themecentre navigation', len(brains))
for index, brain in enumerate(brains):
doc = brain.getObject()
if not INavigationRoot.providedBy(doc):
alsoProvides(doc, INavigationRoot)
pghandler.report(index)
if index % 100 == 0:
transaction.savepoint(optimistic=True)
transaction.savepoint(optimistic=True)
pghandler.finish()
logger.info("Fixing eea.themecentre navigation ... DONE")
def updateIconMetadata(context):
"""Update getIcon metadata column for all core content"""
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon', None)
typesToUpdate = {
'Document': ('document_icon.gif', 'document_icon.png'),
'Event': ('event_icon.gif', 'event_icon.png'),
'File': ('file_icon.gif', 'file_icon.png'),
'Folder': ('folder_icon.gif', 'folder_icon.png'),
'Image': ('image_icon.gif', 'image_icon.png'),
'Link': ('link_icon.gif', 'link_icon.png'),
'News Item': ('newsitem_icon.gif', 'newsitem_icon.png'),
'Topic': ('topic_icon.gif', 'topic_icon.png'),
}
ttool = getToolByName(context, 'portal_types')
empty_icons = []
for name in typesToUpdate.keys():
fti = ttool.get(name)
if fti:
icon_expr = fti.getIconExprObject()
if not icon_expr:
empty_icons.append(name)
brains = search(portal_type=empty_icons, sort_on="path")
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
if not brain_icon:
continue
old_icons = typesToUpdate[brain.portal_type]
if getIconPos is not None:
# if the old icon is a standard icon, we assume no customization
# has taken place and we can simply empty the getIcon metadata
# without loading the object
new_value = ''
if brain_icon not in old_icons:
# Otherwise we need to ask the object
new_value = ''
obj = brain.getObject()
method = getattr(aq_base(obj), 'getIcon', None)
if method is not None:
try:
new_value = obj.getIcon
if callable(new_value):
new_value = new_value()
except ConflictError:
raise
except Exception:
new_value = ''
if brain_icon != new_value:
rid = brain.getRID()
record = metadata[rid]
new_record = list(record)
new_record[getIconPos] = new_value
metadata[rid] = tuple(new_record)
else:
# If we don't have a standard catalog tool, fall back to the
# official API
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(obj, brain_path, ['id'], True,
pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')
def updateIconMetadata(context):
"""Update getIcon metadata column for all core content"""
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon', None)
typesToUpdate = {
'Document' : ('document_icon.gif', 'document_icon.png'),
'Event' : ('event_icon.gif', 'event_icon.png'),
'File' : ('file_icon.gif', 'file_icon.png'),
'Folder' : ('folder_icon.gif', 'folder_icon.png'),
'Image' : ('image_icon.gif', 'image_icon.png'),
'Link' : ('link_icon.gif', 'link_icon.png'),
'News Item' : ('newsitem_icon.gif', 'newsitem_icon.png'),
'Topic' : ('topic_icon.gif', 'topic_icon.png'),
}
ttool = getToolByName(context, 'portal_types')
empty_icons = []
for name in typesToUpdate.keys():
fti = ttool.get(name)
if fti:
icon_expr = fti.getIconExprObject()
if not icon_expr:
empty_icons.append(name)
brains = search(portal_type=empty_icons, sort_on="path")
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
if not brain_icon:
continue
old_icons = typesToUpdate[brain.portal_type]
if getIconPos is not None:
# if the old icon is a standard icon, we assume no customization
# has taken place and we can simply empty the getIcon metadata
# without loading the object
new_value = ''
if brain_icon not in old_icons:
# Otherwise we need to ask the object
new_value = ''
obj = brain.getObject()
method = getattr(aq_base(obj), 'getIcon', None)
if method is not None:
try:
new_value = obj.getIcon
if callable(new_value):
new_value = new_value()
except ConflictError:
raise
except Exception:
new_value = ''
if brain_icon != new_value:
rid = brain.getRID()
record = metadata[rid]
new_record = list(record)
new_record[getIconPos] = new_value
metadata[rid] = tuple(new_record)
else:
# If we don't have a standard catalog tool, fall back to the
# official API
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')