def uidDuplications(self):
pc = getToolByName(self, 'portal_catalog')
ptool = getToolByName(self, 'portal_url')
pghandler = ZLogHandler(1000)
portal = ptool.getPortalObject()
basecat = pc._catalog
index = basecat.indexes['UID']
size = len(index._index.keys())
pghandler.init('Remove stale items from catalog', size)
cnt = 0
for uid, rid in index._index.items():
path = basecat.paths[rid]
pghandler.report(cnt)
cnt += 1
try:
obj = portal.restrictedTraverse(path)
except:
# print "could not get obj for path %s, skipping" % path
# we ignore this for the moment
continue
parent = aq_parent(obj)
ids = [x['id'] for x in parent._objects]
if obj.getId() not in ids and obj.getId() not in parent.objectIds():
print "path %s not in parent " % path
del index._index[uid]
index._length.change(-1)
return "done"
def uidDuplications(self):
pc = getToolByName(self, 'portal_catalog')
ptool = getToolByName(self, 'portal_url')
pghandler = ZLogHandler(1000)
portal = ptool.getPortalObject()
basecat = pc._catalog
index = basecat.indexes['UID']
size = len(index._index.keys())
pghandler.init('Remove stale items from catalog', size)
cnt = 0
for uid, rid in index._index.items():
path = basecat.paths[rid]
pghandler.report(cnt)
cnt += 1
try:
obj = portal.restrictedTraverse(path)
except:
# print "could not get obj for path %s, skipping" % path
# we ignore this for the moment
continue
parent = aq_parent(obj)
ids = [x['id'] for x in parent._objects]
if obj.getId() not in ids and obj.getId() not in parent.objectIds():
print "path %s not in parent " % path
del index._index[uid]
index._length.change(-1)
return "done"
def delColumn(self, name, threshold=10000):
"""Deletes a row from the meta data schema"""
names = list(self.names)
_index = names.index(name)
threshold = threshold if threshold is not None else 10000
if name not in self.schema:
LOG.error('delColumn attempted to delete nonexistent '
'column %s.', str(name))
return
del names[_index]
# rebuild the schema
schema = {}
for i, name in enumerate(names):
schema[name] = i
self.schema = schema
self.names = tuple(names)
# update the brain
self.updateBrains()
# remove the column value from each record
if len(self):
_next_index = _index + 1
pghandler = ZLogHandler(threshold)
pghandler.init('Deleting %s column' % name, len(self))
for i, (key, value) in enumerate(self.data.iteritems()):
pghandler.report(i)
self.data[key] = value[:_index] + value[_next_index:]
pghandler.finish()
def reindexIndexes(self, idxs=[], update_metadata=False, meta_types=[], portal_types=[]):
"""Reindex index including metadata if p_update_metadata=True.
Filter meta_type/portal_type when p_meta_types and p_portal_types are given."""
catalog = api.portal.get_tool('portal_catalog')
paths = catalog._catalog.uids.keys()
pghandler = ZLogHandler(steps=1000)
i = 0
pghandler.info(
'In reindexIndexes, idxs={0}, update_metadata={1}, meta_types={2}, portal_types={3}'.format(
repr(idxs), repr(update_metadata), repr(meta_types), repr(portal_types)))
pghandler.init('reindexIndexes', len(paths))
for p in paths:
i += 1
if pghandler:
pghandler.report(i)
obj = catalog.resolve_path(p)
if obj is None:
logger.error(
'reindexIndex could not resolve an object from the uid %r.' % p)
else:
if (meta_types and obj.meta_type not in meta_types) or \
(portal_types and obj.portal_type not in portal_types):
continue
catalog.catalog_object(
obj, p, idxs=idxs, update_metadata=update_metadata, pghandler=pghandler)
if pghandler:
pghandler.finish()
def update_getIcon(context):
typesToUpdate = {
'Location': ('link_icon.gif', 'link_icon.png'),
'Name': ('document_icon.gif', 'document_icon.png'),
'PleiadesVocabulary': ('folder_icon.gif', 'folder_icon.png'),
'PleiadesVocabularyTerm': ('document_icon.gif', 'document_icon.png'),
}
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon')
brains = search(portal_type=typesToUpdate.keys(), sort_on="path")
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
old_icon, new_icon = typesToUpdate[brain.portal_type]
if brain_icon != new_icon:
rid = brain.getRID()
record = metadata[rid]
new_record = list(record)
new_record[getIconPos] = new_icon
metadata[rid] = tuple(new_record)
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')
def _cleanFTWLabels(self):
"""This fix partial migrations of stored ftw.labels:labeling that are
still PersistentList and not PersistentMapping."""
logger.info("Cleaning ftw.labels wrong annotations...")
brains = self.catalog(object_provides=ILabelSupport.__identifier__)
pghandler = ZLogHandler(steps=100)
pghandler.init('Cleaning ftw.labels wrong annotations...', len(brains))
i = 0
cleaned = 0
for brain in brains:
i += 1
pghandler.report(i)
obj = brain.getObject()
annotations = IAnnotations(obj)
if 'ftw.labels:labeling' in annotations and \
not isinstance(annotations['ftw.labels:labeling'], PersistentMapping):
if annotations['ftw.labels:labeling']:
labeling = ILabeling(obj)
old_values = [label for label in labeling.storage]
del annotations['ftw.labels:labeling']
labeling._storage = None
labeling.update(old_values)
logger.info('In _cleanFTWLabels, cleaned %s' %
brain.getPath())
else:
del annotations['ftw.labels:labeling']
obj.reindexObject(idxs=['labels'])
cleaned += 1
if cleaned:
self.warn(logger,
'In _cleanFTWLabels, cleaned %s element(s)' % cleaned)
pghandler.finish()
logger.info('Done.')
def delColumn(self, name, threshold=10000):
"""Deletes a row from the meta data schema"""
names = list(self.names)
_index = names.index(name)
threshold = threshold if threshold is not None else 10000
if name not in self.schema:
LOG.error(
'delColumn attempted to delete nonexistent '
'column %s.', str(name))
return
del names[_index]
# rebuild the schema
schema = {}
for i, name in enumerate(names):
schema[name] = i
self.schema = schema
self.names = tuple(names)
# update the brain
self.updateBrains()
# remove the column value from each record
if len(self):
_next_index = _index + 1
pghandler = ZLogHandler(threshold)
pghandler.init('Deleting %s column' % name, len(self))
for i, (key, value) in enumerate(self.data.iteritems()):
pghandler.report(i)
self.data[key] = value[:_index] + value[_next_index:]
pghandler.finish()
def reindex_sortable_title(context):
from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
catalog = getToolByName(context, 'portal_catalog')
_catalog = catalog._catalog
indexes = _catalog.indexes
sort_title_index = indexes.get('sortable_title', None)
if sort_title_index is None:
return
from Products.PluginIndexes.FieldIndex import FieldIndex
if not isinstance(sort_title_index, FieldIndex.FieldIndex):
return
change = []
pghandler = ZLogHandler(10000)
logger.info('Analyzing sortable_title index')
pghandler.init('Analyzing sortable_title index', len(sort_title_index))
for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
pghandler.report(i)
if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
try:
keys = rids.keys()
except AttributeError:
change.append(rids)
else:
change.extend(list(keys))
pghandler.finish()
# flake8 complained that reindex_sortable_title is too complex (11).
# So we split it in two.
_update_sortable_title(_catalog, change)
def _migrateLabelForCouncil(self):
"""Field labelForCouncil is replaced by
otherMeetingConfigsClonableToFieldDetailedDescription in College and
detailedDescription in Council."""
logger.info(
'Migrating field "labelForCouncil" in "meeting-config-college"...')
# enable relevant fields in MeetingConfigs
# College we use the "otherMeetingConfigsClonableToFieldLabelForCouncil" field
# Council is correct
college_cfg = self.tool.get('meeting-config-college')
used_attrs = college_cfg.getUsedItemAttributes()
used_attrs = replace_in_list(
used_attrs, "labelForCouncil",
"otherMeetingConfigsClonableToFieldLabelForCouncil")
college_cfg.setUsedItemAttributes(used_attrs)
logger.info('Done.')
logger.info('Migrating field "labelForCouncil" for College items...')
pghandler = ZLogHandler(steps=1000)
brains = self.catalog(portal_type='MeetingItemCollege')
pghandler.init('Migrating field "labelForCouncil" for College items',
len(brains))
i = 0
for brain in brains:
i += 1
pghandler.report(i)
item = brain.getObject()
if not item.fieldIsEmpty('labelForCouncil'):
item.setOtherMeetingConfigsClonableToFieldLabelForCouncil(
item.getRawLabelForCouncil())
item.setLabelForCouncil('')
pghandler.finish()
logger.info('Done.')
def updateIconsInBrains(context, typesToUpdate=None):
"""Update getIcon metadata column in given types.
typesToUpdate must be a dictionary, for example: {
# portal_type: ('old_icon.gif', 'new_icon.png'),
'Document': ('document_icon.gif', 'document_icon.png'),
}
The portal_types must have an empty icon_expr, because that is the
main use case.
"""
if not typesToUpdate:
logger.warn('No typesToUpdate given for updateIconsInBrains.')
return
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon', None)
empty_icons = _types_with_empty_icons(context, typesToUpdate)
brains = search(portal_type=empty_icons, sort_on='path')
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
if not brain_icon:
continue
if getIconPos is not None:
_update_icon_in_single_brain(brain, typesToUpdate, getIconPos,
metadata)
else:
# If we don't have a standard catalog tool, fall back to the
# official API
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(obj, brain_path, ['id'], True,
pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')
def updateIconsInBrains(context, typesToUpdate=None):
"""Update getIcon metadata column in given types.
typesToUpdate must be a dictionary, for example: {
# portal_type: ('old_icon.gif', 'new_icon.png'),
'Document': ('document_icon.gif', 'document_icon.png'),
}
The portal_types must have an empty icon_expr, because that is the
main use case.
"""
if not typesToUpdate:
logger.warn('No typesToUpdate given for updateIconsInBrains.')
return
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon', None)
empty_icons = _types_with_empty_icons(context, typesToUpdate)
brains = search(portal_type=empty_icons, sort_on='path')
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
if not brain_icon:
continue
if getIconPos is not None:
_update_icon_in_single_brain(
brain, typesToUpdate, getIconPos, metadata)
else:
# If we don't have a standard catalog tool, fall back to the
# official API
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(
obj, brain_path, ['id'], True, pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')
def _migrate_categories_to_groups_in_charge_field(self):
logger.info("Migrating Category to Groups in charge field")
catalog = api.portal.get_tool("portal_catalog")
own_org = get_own_organization()
# First we create the appropriate groupsInCharge
logger.info("Adapting MeetingConfig...")
cfg = catalog(portal_type="MeetingConfig",
id=self.meeting_config_id)[0].getObject()
categories = cfg.categories
for cat_id, category in categories.contentItems():
if cat_id not in own_org.objectIds():
data = {
"id": cat_id,
"title": category.Title(),
"description": category.Description()
}
api.content.create(container=own_org,
type="organization",
**data)
new_org_uid = org_id_to_uid(cat_id)
enabled_plonegroup_org_uids = api.portal.get_registry_record(
ORGANIZATIONS_REGISTRY)
if new_org_uid not in enabled_plonegroup_org_uids:
api.portal.set_registry_record(
ORGANIZATIONS_REGISTRY,
enabled_plonegroup_org_uids + [new_org_uid])
ordered_group_in_charge_uids = cfg.getOrderedGroupsInCharge()
if new_org_uid not in ordered_group_in_charge_uids:
cfg.setOrderedGroupsInCharge(ordered_group_in_charge_uids +
(new_org_uid, ))
category.enabled = False
# Next we migrate the category to groupsInCharge on MeetingItems
brains = catalog(portal_type=(
cfg.getItemTypeName(),
cfg.getItemTypeName(configType="MeetingItemRecurring"),
cfg.getItemTypeName(configType="MeetingItemTemplate"),
))
pghandler = ZLogHandler(steps=1000)
pghandler.init("Adapting items...", len(brains))
for i, brain in enumerate(brains):
pghandler.report(i)
item = brain.getObject()
if item.getCategory() != u"_none_":
item.setGroupsInCharge([org_id_to_uid(item.getCategory())])
item.setCategory(None)
item.reindexObject(idxs=["getGroupsInCharge", "getCategory"])
pghandler.finish()
logger.info("Done migrating Categories to Groups in charge field")
def _migrateMeetingCategoryToDX(self):
'''Migrate from AT MeetingCategory to DX meetingcategory.'''
logger.info('Migrating MeetingCategory from AT to DX...')
# update item classifier
# migrate references
pghandler = ZLogHandler(steps=100)
brains = self.portal.reference_catalog(
relationship='ItemClassification')
pghandler.init('Updating field MeetingItem.classifier...', len(brains))
pghandler.info('Updating field MeetingItem.classifier...')
i = 0
for brain in brains:
i += 1
pghandler.report(i)
relation = brain.getObject()
item = relation.getSourceObject()
classifier = relation.getTargetObject()
item.setClassifier(classifier.getId())
item.reindexObject(idxs=['getRawClassifier'])
# deleteReferences in a second phase
for brain in brains:
relation = brain.getObject()
item = relation.getSourceObject()
item.deleteReferences('ItemClassification')
pghandler.finish()
logger.info('Migrating categories and classifiers in configuration...')
# make sure new portal_type meetingcategory is installed
self.ps.runImportStepFromProfile(
'profile-Products.PloneMeeting:default', 'typeinfo')
# make sure no workflow used for meetingcategory
self.wfTool.setChainForPortalTypes(('meetingcategory', ), ('', ))
# adapt allowed_types for each MeetingConfig.categories/classifiers folders
for cfg in self.tool.objectValues('MeetingConfig'):
for folder_id in ('categories', 'classifiers'):
constrain = IConstrainTypes(getattr(cfg, folder_id))
constrain.setConstrainTypesMode(1)
allowedTypes = ['meetingcategory']
constrain.setLocallyAllowedTypes(allowedTypes)
constrain.setImmediatelyAddableTypes(allowedTypes)
# migrate to DX
pac_migrate(self.portal, MeetingCategoryMigrator)
self.removeUnusedPortalTypes(portal_types=['MeetingCategory'])
# add meetingcategory to types_not_searched
props = api.portal.get_tool('portal_properties').site_properties
nsTypes = props.getProperty('types_not_searched')
if 'meetingcategory' not in nsTypes:
nsTypes = list(nsTypes)
# MeetingCategory was removed by removeUnusedPortalTypes
nsTypes.append('meetingcategory')
props.manage_changeProperties(types_not_searched=tuple(nsTypes))
logger.info('Done.')
def _umarkCreationFlagForEveryItems(self):
""" """
brains = self.catalog(meta_type='MeetingItem')
pghandler = ZLogHandler(steps=100)
pghandler.init('Cleaning ftw.labels wrong annotations...', len(brains))
pghandler.info("Unmarking _at_creation_flag for every items...")
i = 0
for brain in brains:
i += 1
pghandler.report(i)
item = brain.getObject()
item.unmarkCreationFlag()
pghandler.finish()
logger.info('Done.')
def _removeMeetingItemGroupedItemsNumAttribute(self):
"""This field was supposed to be used but will not...
Remove the MeetingItem.groupedItemsNum attribute on every existing items."""
logger.info('Removing attribute "groupedItemsNum" from every items...')
brains = self.catalog(meta_type="MeetingItem")
pghandler = ZLogHandler(steps=1000)
pghandler.init('Working', len(brains))
i = 0
for brain in brains:
i += 1
pghandler.report(i)
item = brain.getObject()
safe_delattr(item, "groupedItemsNum")
pghandler.finish()
logger.info('Done.')
def refresh_getIcon_catalog_metadata(context):
"""
get_icon redefined: now boolean:
needs to update metadata
true if item is type image or has image field (named 'image')
e.g. leadimage
see https://github.com/plone/Products.CMFPlone/issues/1226
"""
# Attention, this relies heavily on catalog internals.
# get the more visible zcatalog
zcatalog = getToolByName(context, 'portal_catalog')
# get the more hidden, inner (real) catalog implementation
catalog = zcatalog._catalog
try:
# Get the getIcon index value and
# check if there is a getIcon at all, this may not exist in some
# customizations, who knows, but always exists in default Plone
metadata_index = catalog.names.index('getIcon')
except ValueError:
logger.info('`getIcon` not in metadata, skip upgrade step')
return
cnt = 0
# search whole catalog
results = zcatalog.unrestrictedSearchResults()
num_objects = len(results)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
for brain in results:
pghandler.report(cnt)
# First get the new value
try:
obj = brain._unrestrictedGetObject()
except (AttributeError, KeyError, TypeError, NotFound):
continue
new_value = bool(getattr(aq_base(obj), 'image', False))
# We can now update the record with the new getIcon value
record = list(catalog.data[brain.getRID()])
record[metadata_index] = new_value
catalog.data[brain.getRID()] = tuple(record)
cnt += 1 # we are curious
# done
pghandler.finish()
logger.info('Reindexed `getIcon` for %s items', str(cnt))
def refresh_getIcon_catalog_metadata(context):
"""
get_icon redefined: now boolean:
needs to update metadata
true if item is type image or has image field (named 'image')
e.g. leadimage
see https://github.com/plone/Products.CMFPlone/issues/1226
"""
# Attention, this relies heavily on catalog internals.
# get the more visible zcatalog
zcatalog = getToolByName(context, 'portal_catalog')
# get the more hidden, inner (real) catalog implementation
catalog = zcatalog._catalog
try:
# Get the getIcon index value and
# check if there is a getIcon at all, this may not exist in some
# customizations, who knows, but always exists in default Plone
metadata_index = catalog.names.index('getIcon')
except ValueError:
logger.info('`getIcon` not in metadata, skip upgrade step')
return
cnt = 0
# search whole catalog
results = zcatalog.unrestrictedSearchResults()
num_objects = len(results)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
for brain in results:
pghandler.report(cnt)
# First get the new value
try:
obj = brain._unrestrictedGetObject()
except (AttributeError, KeyError, TypeError, NotFound):
continue
new_value = bool(getattr(aq_base(obj), 'image', False))
# We can now update the record with the new getIcon value
record = list(catalog.data[brain.getRID()])
record[metadata_index] = new_value
catalog.data[brain.getRID()] = tuple(record)
cnt += 1 # we are curious
# done
pghandler.finish()
logger.info('Reindexed `getIcon` for %s items' % str(cnt))
def cleanup(self):
cat = getToolByName(self, "portal_catalog")
purl = getToolByName(self, "portal_url")
portal = purl.getPortalObject()
zcat = cat._catalog
pghandler = ZLogHandler(1000)
size = len(zcat.uids)
pghandler.init('Remove stale items from catalog', size)
cnt = removed = 0
for path in zcat.uids.keys():
pghandler.report(cnt)
cnt += 1
try:
obj = portal.restrictedTraverse(path)
except:
# XXX what to do?
log.warning("no object found for %s" % path)
cat.uncatalog_object(path)
removed += 1
log.info('Kill old path: %s' % path)
else:
if "/".join(obj.getPhysicalPath()) != path:
# saved under an old path! kill it
cat.uncatalog_object(path)
#index = zcat.uids[path]
#del zcat.uids[path]
#del zcat.paths[index]
#zcat._length.change(-1)
removed += 1
log.info('Kill old path: %s' % path)
else:
parent = aq_parent(obj)
ids = [x['id'] for x in parent._objects]
if obj.getId() not in ids and obj.getId(
) not in parent.objectIds():
cat.uncatalog_object(path)
removed += 1
log.info(
'ID not found as a child of current parent, kill it: %s'
% path)
pghandler.finish()
log.info('Finished with the catalog, removed a total of %d items' %
removed)
log.info("Length: %d, len(uids): %d, len(paths): %d" %
(zcat._length.value, len(zcat.uids), len(zcat.paths)))
transaction.commit()
def _update_sortable_title(catalog, change):
update_metadata = 'sortable_title' in catalog.schema
pghandler = ZLogHandler(1000)
logger.info('Updating sortable_title index')
pghandler.init('Updating sortable_title index', len(change))
for i, rid in enumerate(change):
pghandler.report(i)
brain = catalog[rid]
try:
obj = brain.getObject()
except (AttributeError, KeyError):
continue
if update_metadata:
obj.reindexObject()
else:
obj.reindexObject(idxs=['sortable_title'])
pghandler.finish()
logger.info('Updated `sortable_title` index.')
def reindex_mime_type(context):
# Attention, this relies heavily on catalog internals.
# get the more visible zcatalog
zcatalog = getToolByName(context, 'portal_catalog')
# get the more hidden, inner (real) catalog implementation
catalog = zcatalog._catalog
try:
metadata_index = catalog.names.index('mime_type')
except ValueError:
logger.info('`mime_type` not in metadata, skip upgrade step')
return
# we are interested in dexterity and archtetype images and files:
ifaces = ['plone.app.contenttypes.interfaces.IImage',
'plone.app.contenttypes.interfaces.IFile',
'Products.ATContentTypes.interfaces.file.IFileContent',
'Products.ATContentTypes.interfaces.image.IImageContent']
cnt = 0
results = zcatalog.unrestrictedSearchResults(object_provides=ifaces)
num_objects = len(results)
pghandler = ZLogHandler(1000)
pghandler.init('Updating mime_type metadata', num_objects)
for brain in results:
pghandler.report(cnt)
try:
obj = brain._unrestrictedGetObject()
except (AttributeError, KeyError, TypeError, NotFound):
continue
if not obj:
continue
# First get the new value:
new_value = ''
try: # Dexterity
new_value = obj.content_type()
except TypeError: # Archetypes
new_value = obj.content_type
except AttributeError:
continue
# We can now update the record with the new mime_type value
record = list(catalog.data[brain.getRID()])
record[metadata_index] = new_value
catalog.data[brain.getRID()] = tuple(record)
cnt += 1
pghandler.finish()
logger.info('Reindexed `mime_type` for %s items', str(cnt))
def reindex_mime_type(context):
# Attention, this relies heavily on catalog internals.
# get the more visible zcatalog
zcatalog = getToolByName(context, 'portal_catalog')
# get the more hidden, inner (real) catalog implementation
catalog = zcatalog._catalog
try:
metadata_index = catalog.names.index('mime_type')
except ValueError:
logger.info('`mime_type` not in metadata, skip upgrade step')
return
# we are interested in dexterity and archtetype images and files:
ifaces = ['plone.app.contenttypes.interfaces.IImage',
'plone.app.contenttypes.interfaces.IFile',
'Products.ATContentTypes.interfaces.file.IFileContent',
'Products.ATContentTypes.interfaces.image.IImageContent']
cnt = 0
results = zcatalog.unrestrictedSearchResults(object_provides=ifaces)
num_objects = len(results)
pghandler = ZLogHandler(1000)
pghandler.init('Updating mime_type metadata', num_objects)
for brain in results:
pghandler.report(cnt)
try:
obj = brain._unrestrictedGetObject()
except (AttributeError, KeyError, TypeError, NotFound):
continue
if not obj:
continue
# First get the new value:
new_value = ''
try: # Dexterity
new_value = obj.content_type()
except TypeError: # Archetypes
new_value = obj.content_type
except AttributeError:
continue
# We can now update the record with the new mime_type value
record = list(catalog.data[brain.getRID()])
record[metadata_index] = new_value
catalog.data[brain.getRID()] = tuple(record)
cnt += 1
pghandler.finish()
logger.info('Reindexed `mime_type` for %s items' % str(cnt))
def cleanup(self):
cat = getToolByName(self, "portal_catalog")
purl = getToolByName(self, "portal_url")
portal = purl.getPortalObject()
zcat = cat._catalog
pghandler = ZLogHandler(1000)
size = len(zcat.uids)
pghandler.init('Remove stale items from catalog', size)
cnt = removed = 0
for path in zcat.uids.keys():
pghandler.report(cnt)
cnt += 1
try:
obj = portal.restrictedTraverse(path)
except:
# XXX what to do?
log.warning("no object found for %s" % path)
cat.uncatalog_object(path)
removed += 1
log.info('Kill old path: %s' % path)
else:
if "/".join(obj.getPhysicalPath()) != path:
# saved under an old path! kill it
cat.uncatalog_object(path)
#index = zcat.uids[path]
#del zcat.uids[path]
#del zcat.paths[index]
#zcat._length.change(-1)
removed += 1
log.info('Kill old path: %s' % path)
else:
parent = aq_parent(obj)
ids = [x['id'] for x in parent._objects]
if obj.getId() not in ids and obj.getId() not in parent.objectIds():
cat.uncatalog_object(path)
removed += 1
log.info('ID not found as a child of current parent, kill it: %s' % path)
pghandler.finish()
log.info('Finished with the catalog, removed a total of %d items' % removed)
log.info("Length: %d, len(uids): %d, len(paths): %d" % (zcat._length.value,
len(zcat.uids), len(zcat.paths)))
transaction.commit()
def reindex_sortable_title(context):
from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
catalog = getToolByName(context, "portal_catalog")
_catalog = catalog._catalog
indexes = _catalog.indexes
sort_title_index = indexes.get("sortable_title", None)
if sort_title_index is None:
return
from Products.PluginIndexes.FieldIndex import FieldIndex
if not isinstance(sort_title_index, FieldIndex.FieldIndex):
return
change = []
pghandler = ZLogHandler(10000)
logger.info("Analyzing sortable_title index")
pghandler.init("Analyzing sortable_title index", len(sort_title_index))
for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
pghandler.report(i)
if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
if hasattr(rids, "keys"):
change.extend(list(rids.keys()))
else:
change.append(rids)
pghandler.finish()
update_metadata = "sortable_title" in _catalog.schema
pghandler = ZLogHandler(1000)
logger.info("Updating sortable_title index")
pghandler.init("Updating sortable_title index", len(change))
for i, rid in enumerate(change):
pghandler.report(i)
brain = _catalog[rid]
try:
obj = brain.getObject()
except (AttributeError, KeyError):
continue
if update_metadata:
obj.reindexObject()
else:
obj.reindexObject(idxs=["sortable_title"])
pghandler.finish()
logger.info("Updated `sortable_title` index.")
def addColumn(self, name, default_value=None, threshold=10000):
"""Adds a row to the meta data schema"""
schema = self.schema
names = list(self.names)
threshold = threshold if threshold is not None else 10000
if name != name.strip():
# Someone could have mistakenly added a space at the end
# of the input field.
LOG.warning('stripped space from new column %r -> %r', name,
name.strip())
name = name.strip()
if name in schema:
raise CatalogError('The column %s already exists' % name)
if name[0] == '_':
raise CatalogError('Cannot cache fields beginning with "_"')
values = schema.values()
if values:
schema[name] = max(values) + 1
else:
schema[name] = 0
names.append(name)
if default_value in (None, ''):
default_value = MV
if len(self):
pghandler = ZLogHandler(threshold)
pghandler.init('Adding %s column' % name, len(self))
for i, (key, value) in enumerate(self.data.iteritems()):
pghandler.report(i)
self.data[key] = value + (default_value, )
pghandler.finish()
self.names = tuple(names)
self.schema = schema
# new column? update the brain
self.updateBrains()
def addColumn(self, name, default_value=None, threshold=10000):
"""Adds a row to the meta data schema"""
schema = self.schema
names = list(self.names)
threshold = threshold if threshold is not None else 10000
if name != name.strip():
# Someone could have mistakenly added a space at the end
# of the input field.
LOG.warning('stripped space from new column %r -> %r', name,
name.strip())
name = name.strip()
if name in schema:
raise CatalogError('The column %s already exists' % name)
if name[0] == '_':
raise CatalogError('Cannot cache fields beginning with "_"')
values = schema.values()
if values:
schema[name] = max(values) + 1
else:
schema[name] = 0
names.append(name)
if default_value in (None, ''):
default_value = MV
if len(self):
pghandler = ZLogHandler(threshold)
pghandler.init('Adding %s column' % name, len(self))
for i, (key, value) in enumerate(self.data.iteritems()):
pghandler.report(i)
self.data[key] = value + (default_value, )
pghandler.finish()
self.names = tuple(names)
self.schema = schema
# new column? update the brain
self.updateBrains()
def reindexIndexesFor(self, idxs=[], **query):
""" Reindex p_idxs on objects of given p_portal_types. """
catalog = api.portal.get_tool('portal_catalog')
brains = catalog(**query)
pghandler = ZLogHandler(steps=1000)
len_brains = len(brains)
pghandler.info(
'In reindexIndexesFor, reindexing indexes "{0}" on "{1}" objects ({2})...'.format(
', '.join(idxs) or '*',
len(brains),
str(query)))
pghandler.init('reindexIndexesFor', len_brains)
i = 0
for brain in brains:
i += 1
pghandler.report(i)
obj = brain.getObject()
obj.reindexObject(idxs=idxs)
pghandler.finish()
logger.info('Done.')
def reindex_sortable_title(context):
from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
catalog = getToolByName(context, 'portal_catalog')
_catalog = catalog._catalog
indexes = _catalog.indexes
sort_title_index = indexes.get('sortable_title', None)
if sort_title_index is None:
return
from Products.PluginIndexes.FieldIndex import FieldIndex
if not isinstance(sort_title_index, FieldIndex.FieldIndex):
return
change = []
pghandler = ZLogHandler(10000)
logger.info('Analyzing sortable_title index')
pghandler.init('Analyzing sortable_title index', len(sort_title_index))
for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
pghandler.report(i)
if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
if hasattr(rids, 'keys'):
change.extend(list(rids.keys()))
else:
change.append(rids)
pghandler.finish()
update_metadata = 'sortable_title' in _catalog.schema
pghandler = ZLogHandler(1000)
logger.info('Updating sortable_title index')
pghandler.init('Updating sortable_title index', len(change))
for i, rid in enumerate(change):
pghandler.report(i)
brain = _catalog[rid]
try:
obj = brain.getObject()
except (AttributeError, KeyError):
continue
if update_metadata:
obj.reindexObject()
else:
obj.reindexObject(idxs=['sortable_title'])
pghandler.finish()
logger.info('Updated `sortable_title` index.')
def agenda_item_project_set_summary_html(context):
"""Set the mimetype of the summary to text/html.
Otherwise when editing you get a bare textarea and cannot save it.
"""
catalog = getToolByName(context, 'portal_catalog')
brains = catalog(portal_type='AgendaItemProject')
num_objects = len(brains)
logger.info("Updating agenda item summaries.")
fixed = 0
pghandler = ZLogHandler(100)
pghandler.init('Setting mimetype html for agenda item summaries',
num_objects)
for index, brain in enumerate(brains):
pghandler.report(index)
obj = brain.getObject()
if obj.summary.mimetype != 'text/html':
obj.summary.mimetype = 'text/html'
fixed += 1
pghandler.finish()
logger.info("Have set summary mimetype to text/html for {0}/{1} agenda "
"items.".format(fixed, len(brains)))
def update_catalog_metadata(context, types=None):
"""Update catalog metadata.
Adapted from updateIconMetadata in plone.app.upgrade v40/betas.py.
"""
catalog = getToolByName(context, 'portal_catalog')
search = catalog.unrestrictedSearchResults
if types is None:
logger.info('Updating catalog metadata.')
brains = search(sort_on="path")
else:
if isinstance(types, basestring):
types = [types]
logger.info('Updating catalog metadata for %s.', ', '.join(types))
brains = search(portal_type=types, sort_on="path")
num_objects = len(brains)
# Yes, this logs quite often, but I just know I am going to thank
# myself for that... Feedback about progress is good for my heart.
# [Maurits]
pghandler = ZLogHandler(10)
pghandler.init('Updating catalog metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Done updating catalog metadata.')
def fix_themes_navigation(context):
""" Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
objects.
"""
ctool = getToolByName(context, 'portal_catalog')
brains = ctool.unrestrictedSearchResults(
object_provides='eea.themecentre.interfaces.IThemeCentre')
logger.info("Fixing eea.themecentre navigation ...")
pghandler = ZLogHandler(100)
pghandler.init('Fixing eea.themecentre navigation', len(brains))
for index, brain in enumerate(brains):
doc = brain.getObject()
if not INavigationRoot.providedBy(doc):
alsoProvides(doc, INavigationRoot)
pghandler.report(index)
if index % 100 == 0:
transaction.savepoint(optimistic=True)
transaction.savepoint(optimistic=True)
pghandler.finish()
logger.info("Fixing eea.themecentre navigation ... DONE")
def fix_themes_navigation(context):
""" Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
objects.
"""
ctool = getToolByName(context, 'portal_catalog')
brains = ctool.unrestrictedSearchResults(
object_provides='eea.themecentre.interfaces.IThemeCentre')
logger.info("Fixing eea.themecentre navigation ...")
pghandler = ZLogHandler(100)
pghandler.init('Fixing eea.themecentre navigation', len(brains))
for index, brain in enumerate(brains):
doc = brain.getObject()
if not INavigationRoot.providedBy(doc):
alsoProvides(doc, INavigationRoot)
pghandler.report(index)
if index % 100 == 0:
transaction.savepoint(optimistic=True)
transaction.savepoint(optimistic=True)
pghandler.finish()
logger.info("Fixing eea.themecentre navigation ... DONE")
def updateIconMetadata(context):
"""Update getIcon metadata column for all core content"""
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon', None)
typesToUpdate = {
'Document': ('document_icon.gif', 'document_icon.png'),
'Event': ('event_icon.gif', 'event_icon.png'),
'File': ('file_icon.gif', 'file_icon.png'),
'Folder': ('folder_icon.gif', 'folder_icon.png'),
'Image': ('image_icon.gif', 'image_icon.png'),
'Link': ('link_icon.gif', 'link_icon.png'),
'News Item': ('newsitem_icon.gif', 'newsitem_icon.png'),
'Topic': ('topic_icon.gif', 'topic_icon.png'),
}
ttool = getToolByName(context, 'portal_types')
empty_icons = []
for name in typesToUpdate.keys():
fti = ttool.get(name)
if fti:
icon_expr = fti.getIconExprObject()
if not icon_expr:
empty_icons.append(name)
brains = search(portal_type=empty_icons, sort_on="path")
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
if not brain_icon:
continue
old_icons = typesToUpdate[brain.portal_type]
if getIconPos is not None:
# if the old icon is a standard icon, we assume no customization
# has taken place and we can simply empty the getIcon metadata
# without loading the object
new_value = ''
if brain_icon not in old_icons:
# Otherwise we need to ask the object
new_value = ''
obj = brain.getObject()
method = getattr(aq_base(obj), 'getIcon', None)
if method is not None:
try:
new_value = obj.getIcon
if callable(new_value):
new_value = new_value()
except ConflictError:
raise
except Exception:
new_value = ''
if brain_icon != new_value:
rid = brain.getRID()
record = metadata[rid]
new_record = list(record)
new_record[getIconPos] = new_value
metadata[rid] = tuple(new_record)
else:
# If we don't have a standard catalog tool, fall back to the
# official API
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(obj, brain_path, ['id'], True,
pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')
def updateIconMetadata(context):
"""Update getIcon metadata column for all core content"""
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon', None)
typesToUpdate = {
'Document' : ('document_icon.gif', 'document_icon.png'),
'Event' : ('event_icon.gif', 'event_icon.png'),
'File' : ('file_icon.gif', 'file_icon.png'),
'Folder' : ('folder_icon.gif', 'folder_icon.png'),
'Image' : ('image_icon.gif', 'image_icon.png'),
'Link' : ('link_icon.gif', 'link_icon.png'),
'News Item' : ('newsitem_icon.gif', 'newsitem_icon.png'),
'Topic' : ('topic_icon.gif', 'topic_icon.png'),
}
ttool = getToolByName(context, 'portal_types')
empty_icons = []
for name in typesToUpdate.keys():
fti = ttool.get(name)
if fti:
icon_expr = fti.getIconExprObject()
if not icon_expr:
empty_icons.append(name)
brains = search(portal_type=empty_icons, sort_on="path")
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
if not brain_icon:
continue
old_icons = typesToUpdate[brain.portal_type]
if getIconPos is not None:
# if the old icon is a standard icon, we assume no customization
# has taken place and we can simply empty the getIcon metadata
# without loading the object
new_value = ''
if brain_icon not in old_icons:
# Otherwise we need to ask the object
new_value = ''
obj = brain.getObject()
method = getattr(aq_base(obj), 'getIcon', None)
if method is not None:
try:
new_value = obj.getIcon
if callable(new_value):
new_value = new_value()
except ConflictError:
raise
except Exception:
new_value = ''
if brain_icon != new_value:
rid = brain.getRID()
record = metadata[rid]
new_record = list(record)
new_record[getIconPos] = new_value
metadata[rid] = tuple(new_record)
else:
# If we don't have a standard catalog tool, fall back to the
# official API
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')