def reindexIndexes(self, idxs=[], update_metadata=False, meta_types=[], portal_types=[]):
"""Reindex index including metadata if p_update_metadata=True.
Filter meta_type/portal_type when p_meta_types and p_portal_types are given."""
catalog = api.portal.get_tool('portal_catalog')
paths = catalog._catalog.uids.keys()
pghandler = ZLogHandler(steps=1000)
i = 0
pghandler.info(
'In reindexIndexes, idxs={0}, update_metadata={1}, meta_types={2}, portal_types={3}'.format(
repr(idxs), repr(update_metadata), repr(meta_types), repr(portal_types)))
pghandler.init('reindexIndexes', len(paths))
for p in paths:
i += 1
if pghandler:
pghandler.report(i)
obj = catalog.resolve_path(p)
if obj is None:
logger.error(
'reindexIndex could not resolve an object from the uid %r.' % p)
else:
if (meta_types and obj.meta_type not in meta_types) or \
(portal_types and obj.portal_type not in portal_types):
continue
catalog.catalog_object(
obj, p, idxs=idxs, update_metadata=update_metadata, pghandler=pghandler)
if pghandler:
pghandler.finish()
def uidDuplications(self):
pc = getToolByName(self, 'portal_catalog')
ptool = getToolByName(self, 'portal_url')
pghandler = ZLogHandler(1000)
portal = ptool.getPortalObject()
basecat = pc._catalog
index = basecat.indexes['UID']
size = len(index._index.keys())
pghandler.init('Remove stale items from catalog', size)
cnt = 0
for uid, rid in index._index.items():
path = basecat.paths[rid]
pghandler.report(cnt)
cnt += 1
try:
obj = portal.restrictedTraverse(path)
except:
# print "could not get obj for path %s, skipping" % path
# we ignore this for the moment
continue
parent = aq_parent(obj)
ids = [x['id'] for x in parent._objects]
if obj.getId() not in ids and obj.getId() not in parent.objectIds():
print "path %s not in parent " % path
del index._index[uid]
index._length.change(-1)
return "done"
def _migrateMeetingCategoryToDX(self):
'''Migrate from AT MeetingCategory to DX meetingcategory.'''
logger.info('Migrating MeetingCategory from AT to DX...')
# update item classifier
# migrate references
pghandler = ZLogHandler(steps=100)
brains = self.portal.reference_catalog(
relationship='ItemClassification')
pghandler.init('Updating field MeetingItem.classifier...', len(brains))
pghandler.info('Updating field MeetingItem.classifier...')
i = 0
for brain in brains:
i += 1
pghandler.report(i)
relation = brain.getObject()
item = relation.getSourceObject()
classifier = relation.getTargetObject()
item.setClassifier(classifier.getId())
item.reindexObject(idxs=['getRawClassifier'])
# deleteReferences in a second phase
for brain in brains:
relation = brain.getObject()
item = relation.getSourceObject()
item.deleteReferences('ItemClassification')
pghandler.finish()
logger.info('Migrating categories and classifiers in configuration...')
# make sure new portal_type meetingcategory is installed
self.ps.runImportStepFromProfile(
'profile-Products.PloneMeeting:default', 'typeinfo')
# make sure no workflow used for meetingcategory
self.wfTool.setChainForPortalTypes(('meetingcategory', ), ('', ))
# adapt allowed_types for each MeetingConfig.categories/classifiers folders
for cfg in self.tool.objectValues('MeetingConfig'):
for folder_id in ('categories', 'classifiers'):
constrain = IConstrainTypes(getattr(cfg, folder_id))
constrain.setConstrainTypesMode(1)
allowedTypes = ['meetingcategory']
constrain.setLocallyAllowedTypes(allowedTypes)
constrain.setImmediatelyAddableTypes(allowedTypes)
# migrate to DX
pac_migrate(self.portal, MeetingCategoryMigrator)
self.removeUnusedPortalTypes(portal_types=['MeetingCategory'])
# add meetingcategory to types_not_searched
props = api.portal.get_tool('portal_properties').site_properties
nsTypes = props.getProperty('types_not_searched')
if 'meetingcategory' not in nsTypes:
nsTypes = list(nsTypes)
# MeetingCategory was removed by removeUnusedPortalTypes
nsTypes.append('meetingcategory')
props.manage_changeProperties(types_not_searched=tuple(nsTypes))
logger.info('Done.')
def delColumn(self, name, threshold=10000):
"""Deletes a row from the meta data schema"""
names = list(self.names)
_index = names.index(name)
threshold = threshold if threshold is not None else 10000
if name not in self.schema:
LOG.error(
'delColumn attempted to delete nonexistent '
'column %s.', str(name))
return
del names[_index]
# rebuild the schema
schema = {}
for i, name in enumerate(names):
schema[name] = i
self.schema = schema
self.names = tuple(names)
# update the brain
self.updateBrains()
# remove the column value from each record
if len(self):
_next_index = _index + 1
pghandler = ZLogHandler(threshold)
pghandler.init('Deleting %s column' % name, len(self))
for i, (key, value) in enumerate(self.data.iteritems()):
pghandler.report(i)
self.data[key] = value[:_index] + value[_next_index:]
pghandler.finish()
def _cleanFTWLabels(self):
"""This fix partial migrations of stored ftw.labels:labeling that are
still PersistentList and not PersistentMapping."""
logger.info("Cleaning ftw.labels wrong annotations...")
brains = self.catalog(object_provides=ILabelSupport.__identifier__)
pghandler = ZLogHandler(steps=100)
pghandler.init('Cleaning ftw.labels wrong annotations...', len(brains))
i = 0
cleaned = 0
for brain in brains:
i += 1
pghandler.report(i)
obj = brain.getObject()
annotations = IAnnotations(obj)
if 'ftw.labels:labeling' in annotations and \
not isinstance(annotations['ftw.labels:labeling'], PersistentMapping):
if annotations['ftw.labels:labeling']:
labeling = ILabeling(obj)
old_values = [label for label in labeling.storage]
del annotations['ftw.labels:labeling']
labeling._storage = None
labeling.update(old_values)
logger.info('In _cleanFTWLabels, cleaned %s' %
brain.getPath())
else:
del annotations['ftw.labels:labeling']
obj.reindexObject(idxs=['labels'])
cleaned += 1
if cleaned:
self.warn(logger,
'In _cleanFTWLabels, cleaned %s element(s)' % cleaned)
pghandler.finish()
logger.info('Done.')
def reindex_sortable_title(context):
from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
catalog = getToolByName(context, 'portal_catalog')
_catalog = catalog._catalog
indexes = _catalog.indexes
sort_title_index = indexes.get('sortable_title', None)
if sort_title_index is None:
return
from Products.PluginIndexes.FieldIndex import FieldIndex
if not isinstance(sort_title_index, FieldIndex.FieldIndex):
return
change = []
pghandler = ZLogHandler(10000)
logger.info('Analyzing sortable_title index')
pghandler.init('Analyzing sortable_title index', len(sort_title_index))
for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
pghandler.report(i)
if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
try:
keys = rids.keys()
except AttributeError:
change.append(rids)
else:
change.extend(list(keys))
pghandler.finish()
# flake8 complained that reindex_sortable_title is too complex (11).
# So we split it in two.
_update_sortable_title(_catalog, change)
def _migrateLabelForCouncil(self):
"""Field labelForCouncil is replaced by
otherMeetingConfigsClonableToFieldDetailedDescription in College and
detailedDescription in Council."""
logger.info(
'Migrating field "labelForCouncil" in "meeting-config-college"...')
# enable relevant fields in MeetingConfigs
# College we use the "otherMeetingConfigsClonableToFieldLabelForCouncil" field
# Council is correct
college_cfg = self.tool.get('meeting-config-college')
used_attrs = college_cfg.getUsedItemAttributes()
used_attrs = replace_in_list(
used_attrs, "labelForCouncil",
"otherMeetingConfigsClonableToFieldLabelForCouncil")
college_cfg.setUsedItemAttributes(used_attrs)
logger.info('Done.')
logger.info('Migrating field "labelForCouncil" for College items...')
pghandler = ZLogHandler(steps=1000)
brains = self.catalog(portal_type='MeetingItemCollege')
pghandler.init('Migrating field "labelForCouncil" for College items',
len(brains))
i = 0
for brain in brains:
i += 1
pghandler.report(i)
item = brain.getObject()
if not item.fieldIsEmpty('labelForCouncil'):
item.setOtherMeetingConfigsClonableToFieldLabelForCouncil(
item.getRawLabelForCouncil())
item.setLabelForCouncil('')
pghandler.finish()
logger.info('Done.')
def _umarkCreationFlagForEveryItems(self):
""" """
brains = self.catalog(meta_type='MeetingItem')
pghandler = ZLogHandler(steps=100)
pghandler.init('Cleaning ftw.labels wrong annotations...', len(brains))
pghandler.info("Unmarking _at_creation_flag for every items...")
i = 0
for brain in brains:
i += 1
pghandler.report(i)
item = brain.getObject()
item.unmarkCreationFlag()
pghandler.finish()
logger.info('Done.')
def import_localroles(self, data):
results = 0
total = len(data)
for index, item in enumerate(data, start=1):
obj = api.content.get(UID=item["uuid"])
if not obj:
continue
if item.get("localroles"):
localroles = item["localroles"]
for userid in localroles:
obj.manage_setLocalRoles(userid=userid,
roles=localroles[userid])
logger.debug(u"Set roles on {}: {}".format(
obj.absolute_url(), localroles))
if item.get("block"):
obj.__ac_local_roles_block__ = 1
logger.debug(u"Diable acquisition of local roles on {}".format(
obj.absolute_url()))
if not index % 1000:
logger.info(u"Set local roles on {} ({}%) of {} items".format(
index, round(index / total * 100, 2), total))
results += 1
if results:
logger.info("Reindexing Security")
catalog = api.portal.get_tool("portal_catalog")
pghandler = ZLogHandler(1000)
catalog.reindexIndex("allowedRolesAndUsers",
None,
pghandler=pghandler)
return results
def refreshCatalogs(self):
"""
It reindexes the modified catalogs but, while cleanAndRebuildCatalogs
recatalogs all objects in the database, this method only reindexes over
the already cataloged objects.
If a metacolumn is added it refreshes the catalog, if only a new index
is added, it reindexes only those new indexes.
"""
to_refresh = self.refreshcatalog[:]
to_reindex = self.reindexcatalog.keys()
to_reindex = to_reindex[:]
done = []
# Start reindexing the catalogs with new columns
for catalog_to_refresh in to_refresh:
try:
logger.info('Catalog {0} refreshing started'.format(
catalog_to_refresh))
catalog = getToolByName(self.portal, catalog_to_refresh)
handler = ZLogHandler(self.pgthreshold)
catalog.refreshCatalog(pghandler=handler)
logger.info('Catalog {0} refreshed'.format(catalog_to_refresh))
transaction.commit()
except:
logger.error(
'Unable to refresh catalog {0}'.format(catalog_to_refresh))
raise
done.append(catalog_to_refresh)
# Now the catalogs which only need reindxing
for catalog_to_reindex in to_reindex:
if catalog_to_reindex in done:
continue
try:
logger.info('Catalog {0} reindexing started'.format(
catalog_to_reindex))
catalog = getToolByName(self.portal, catalog_to_reindex)
indexes = self.reindexcatalog[catalog_to_reindex]
handler = ZLogHandler(self.pgthreshold)
catalog.reindexIndex(indexes, None, pghandler=handler)
logger.info('Catalog {0} reindexed'.format(catalog_to_reindex))
transaction.commit()
except:
logger.error(
'Unable to reindex catalog {0}'.format(catalog_to_reindex))
raise
done.append(catalog_to_reindex)
def update_getIcon(context):
typesToUpdate = {
'Location': ('link_icon.gif', 'link_icon.png'),
'Name': ('document_icon.gif', 'document_icon.png'),
'PleiadesVocabulary': ('folder_icon.gif', 'folder_icon.png'),
'PleiadesVocabularyTerm': ('document_icon.gif', 'document_icon.png'),
}
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon')
brains = search(portal_type=typesToUpdate.keys(), sort_on="path")
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
old_icon, new_icon = typesToUpdate[brain.portal_type]
if brain_icon != new_icon:
rid = brain.getRID()
record = metadata[rid]
new_record = list(record)
new_record[getIconPos] = new_icon
metadata[rid] = tuple(new_record)
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')
def delColumn(self, name, threshold=10000):
"""Deletes a row from the meta data schema"""
names = list(self.names)
_index = names.index(name)
threshold = threshold if threshold is not None else 10000
if name not in self.schema:
LOG.error('delColumn attempted to delete nonexistent '
'column %s.', str(name))
return
del names[_index]
# rebuild the schema
schema = {}
for i, name in enumerate(names):
schema[name] = i
self.schema = schema
self.names = tuple(names)
# update the brain
self.updateBrains()
# remove the column value from each record
if len(self):
_next_index = _index + 1
pghandler = ZLogHandler(threshold)
pghandler.init('Deleting %s column' % name, len(self))
for i, (key, value) in enumerate(self.data.iteritems()):
pghandler.report(i)
self.data[key] = value[:_index] + value[_next_index:]
pghandler.finish()
def updateIconsInBrains(context, typesToUpdate=None):
"""Update getIcon metadata column in given types.
typesToUpdate must be a dictionary, for example: {
# portal_type: ('old_icon.gif', 'new_icon.png'),
'Document': ('document_icon.gif', 'document_icon.png'),
}
The portal_types must have an empty icon_expr, because that is the
main use case.
"""
if not typesToUpdate:
logger.warn('No typesToUpdate given for updateIconsInBrains.')
return
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon', None)
empty_icons = _types_with_empty_icons(context, typesToUpdate)
brains = search(portal_type=empty_icons, sort_on='path')
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
if not brain_icon:
continue
if getIconPos is not None:
_update_icon_in_single_brain(brain, typesToUpdate, getIconPos,
metadata)
else:
# If we don't have a standard catalog tool, fall back to the
# official API
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(obj, brain_path, ['id'], True,
pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')
def reindex_getWorksheetUID(portal):
"""Reindex getWorksheetUID index from analysis_catalog
"""
catalog_name = CATALOG_ANALYSIS_LISTING
logger.info("Reindexing getWorksheetUID for {} ...".format(catalog_name))
handler = ZLogHandler(steps=100)
catalog = api.get_tool(catalog_name)
catalog.reindexIndex("getWorksheetUID", None, pghandler=handler)
commit_transaction(portal)
def _migrate_categories_to_groups_in_charge_field(self):
logger.info("Migrating Category to Groups in charge field")
catalog = api.portal.get_tool("portal_catalog")
own_org = get_own_organization()
# First we create the appropriate groupsInCharge
logger.info("Adapting MeetingConfig...")
cfg = catalog(portal_type="MeetingConfig",
id=self.meeting_config_id)[0].getObject()
categories = cfg.categories
for cat_id, category in categories.contentItems():
if cat_id not in own_org.objectIds():
data = {
"id": cat_id,
"title": category.Title(),
"description": category.Description()
}
api.content.create(container=own_org,
type="organization",
**data)
new_org_uid = org_id_to_uid(cat_id)
enabled_plonegroup_org_uids = api.portal.get_registry_record(
ORGANIZATIONS_REGISTRY)
if new_org_uid not in enabled_plonegroup_org_uids:
api.portal.set_registry_record(
ORGANIZATIONS_REGISTRY,
enabled_plonegroup_org_uids + [new_org_uid])
ordered_group_in_charge_uids = cfg.getOrderedGroupsInCharge()
if new_org_uid not in ordered_group_in_charge_uids:
cfg.setOrderedGroupsInCharge(ordered_group_in_charge_uids +
(new_org_uid, ))
category.enabled = False
# Next we migrate the category to groupsInCharge on MeetingItems
brains = catalog(portal_type=(
cfg.getItemTypeName(),
cfg.getItemTypeName(configType="MeetingItemRecurring"),
cfg.getItemTypeName(configType="MeetingItemTemplate"),
))
pghandler = ZLogHandler(steps=1000)
pghandler.init("Adapting items...", len(brains))
for i, brain in enumerate(brains):
pghandler.report(i)
item = brain.getObject()
if item.getCategory() != u"_none_":
item.setGroupsInCharge([org_id_to_uid(item.getCategory())])
item.setCategory(None)
item.reindexObject(idxs=["getGroupsInCharge", "getCategory"])
pghandler.finish()
logger.info("Done migrating Categories to Groups in charge field")
def rebuild_uid_index(self):
"""Rebuild uid index.
"""
portal_catalog = getToolByName(self.context, 'portal_catalog')
portal_catalog._catalog.clearIndex('UID')
portal_catalog._catalog.reindexIndex('UID',
self.request,
pghandler=ZLogHandler())
IStatusMessage(self.request).addStatusMessage('Rebuilt UID index.',
type='info')
return self.redirect_to_view()
def reindexIndexesFor(self, idxs=[], **query):
""" Reindex p_idxs on objects of given p_portal_types. """
catalog = api.portal.get_tool('portal_catalog')
brains = catalog(**query)
pghandler = ZLogHandler(steps=1000)
len_brains = len(brains)
pghandler.info(
'In reindexIndexesFor, reindexing indexes "{0}" on "{1}" objects ({2})...'.format(
', '.join(idxs) or '*',
len(brains),
str(query)))
pghandler.init('reindexIndexesFor', len_brains)
i = 0
for brain in brains:
i += 1
pghandler.report(i)
obj = brain.getObject()
obj.reindexObject(idxs=idxs)
pghandler.finish()
logger.info('Done.')
def uidDuplications(self):
pc = getToolByName(self, 'portal_catalog')
ptool = getToolByName(self, 'portal_url')
pghandler = ZLogHandler(1000)
portal = ptool.getPortalObject()
basecat = pc._catalog
index = basecat.indexes['UID']
size = len(index._index.keys())
pghandler.init('Remove stale items from catalog', size)
cnt = 0
for uid, rid in index._index.items():
path = basecat.paths[rid]
pghandler.report(cnt)
cnt += 1
try:
obj = portal.restrictedTraverse(path)
except:
# print "could not get obj for path %s, skipping" % path
# we ignore this for the moment
continue
parent = aq_parent(obj)
ids = [x['id'] for x in parent._objects]
if obj.getId() not in ids and obj.getId() not in parent.objectIds():
print "path %s not in parent " % path
del index._index[uid]
index._length.change(-1)
return "done"
def updateIconsInBrains(context, typesToUpdate=None):
"""Update getIcon metadata column in given types.
typesToUpdate must be a dictionary, for example: {
# portal_type: ('old_icon.gif', 'new_icon.png'),
'Document': ('document_icon.gif', 'document_icon.png'),
}
The portal_types must have an empty icon_expr, because that is the
main use case.
"""
if not typesToUpdate:
logger.warn('No typesToUpdate given for updateIconsInBrains.')
return
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon', None)
empty_icons = _types_with_empty_icons(context, typesToUpdate)
brains = search(portal_type=empty_icons, sort_on='path')
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
if not brain_icon:
continue
if getIconPos is not None:
_update_icon_in_single_brain(
brain, typesToUpdate, getIconPos, metadata)
else:
# If we don't have a standard catalog tool, fall back to the
# official API
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(
obj, brain_path, ['id'], True, pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')
def catalog_rebuild_index(self, name):
"""Reindex the ``portal_catalog`` index identified by ``name``.
"""
catalog = self.getToolByName('portal_catalog')
LOG.info("Reindexing index %s" % name)
# pylint: disable=W0212
pgthreshold = catalog._getProgressThreshold() or 100
# pylint: enable=W0212
pghandler = ZLogHandler(pgthreshold)
catalog.reindexIndex(name, None, pghandler=pghandler)
LOG.info("Reindexing index %s DONE" % name)
def add_metadata(portal, catalog_id, column, refresh_catalog=False):
logger.info("Adding '{}' metadata to '{}' ...".format(column, catalog_id))
catalog = api.get_tool(catalog_id)
if column in catalog.schema():
logger.info("Metadata '{}' already in catalog '{}' [SKIP]"
.format(column, catalog_id))
return
catalog.addColumn(column)
if refresh_catalog:
logger.info("Refreshing catalog '{}' ...".format(catalog_id))
handler = ZLogHandler(steps=100)
catalog.refreshCatalog(pghandler=handler)
def _removeMeetingItemGroupedItemsNumAttribute(self):
"""This field was supposed to be used but will not...
Remove the MeetingItem.groupedItemsNum attribute on every existing items."""
logger.info('Removing attribute "groupedItemsNum" from every items...')
brains = self.catalog(meta_type="MeetingItem")
pghandler = ZLogHandler(steps=1000)
pghandler.init('Working', len(brains))
i = 0
for brain in brains:
i += 1
pghandler.report(i)
item = brain.getObject()
safe_delattr(item, "groupedItemsNum")
pghandler.finish()
logger.info('Done.')
def refresh_getIcon_catalog_metadata(context):
"""
get_icon redefined: now boolean:
needs to update metadata
true if item is type image or has image field (named 'image')
e.g. leadimage
see https://github.com/plone/Products.CMFPlone/issues/1226
"""
# Attention, this relies heavily on catalog internals.
# get the more visible zcatalog
zcatalog = getToolByName(context, 'portal_catalog')
# get the more hidden, inner (real) catalog implementation
catalog = zcatalog._catalog
try:
# Get the getIcon index value and
# check if there is a getIcon at all, this may not exist in some
# customizations, who knows, but always exists in default Plone
metadata_index = catalog.names.index('getIcon')
except ValueError:
logger.info('`getIcon` not in metadata, skip upgrade step')
return
cnt = 0
# search whole catalog
results = zcatalog.unrestrictedSearchResults()
num_objects = len(results)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
for brain in results:
pghandler.report(cnt)
# First get the new value
try:
obj = brain._unrestrictedGetObject()
except (AttributeError, KeyError, TypeError, NotFound):
continue
new_value = bool(getattr(aq_base(obj), 'image', False))
# We can now update the record with the new getIcon value
record = list(catalog.data[brain.getRID()])
record[metadata_index] = new_value
catalog.data[brain.getRID()] = tuple(record)
cnt += 1 # we are curious
# done
pghandler.finish()
logger.info('Reindexed `getIcon` for %s items' % str(cnt))
def del_metadata(portal, catalog_id, column, refresh_catalog=False):
logger.info("Removing '{}' metadata from '{}' ..."
.format(column, catalog_id))
catalog = api.get_tool(catalog_id)
if column not in catalog.schema():
logger.info("Metadata '{}' not in catalog '{}' [SKIP]"
.format(column, catalog_id))
return
catalog.delColumn(column)
if refresh_catalog:
logger.info("Refreshing catalog '{}' ...".format(catalog_id))
handler = ZLogHandler(steps=100)
catalog.refreshCatalog(pghandler=handler)
def reindex_sortable_title(portal):
"""Reindex sortable_title from some catalogs
"""
catalogs = [
"bika_catalog",
"bika_setup_catalog",
"portal_catalog",
]
for catalog_name in catalogs:
logger.info("Reindexing sortable_title for {} ...".format(catalog_name))
handler = ZLogHandler(steps=100)
catalog = api.get_tool(catalog_name)
catalog.reindexIndex("sortable_title", None, pghandler=handler)
commit_transaction(portal)
def cleanup(self):
cat = getToolByName(self, "portal_catalog")
purl = getToolByName(self, "portal_url")
portal = purl.getPortalObject()
zcat = cat._catalog
pghandler = ZLogHandler(1000)
size = len(zcat.uids)
pghandler.init('Remove stale items from catalog', size)
cnt = removed = 0
for path in zcat.uids.keys():
pghandler.report(cnt)
cnt += 1
try:
obj = portal.restrictedTraverse(path)
except:
# XXX what to do?
log.warning("no object found for %s" % path)
cat.uncatalog_object(path)
removed += 1
log.info('Kill old path: %s' % path)
else:
if "/".join(obj.getPhysicalPath()) != path:
# saved under an old path! kill it
cat.uncatalog_object(path)
#index = zcat.uids[path]
#del zcat.uids[path]
#del zcat.paths[index]
#zcat._length.change(-1)
removed += 1
log.info('Kill old path: %s' % path)
else:
parent = aq_parent(obj)
ids = [x['id'] for x in parent._objects]
if obj.getId() not in ids and obj.getId(
) not in parent.objectIds():
cat.uncatalog_object(path)
removed += 1
log.info(
'ID not found as a child of current parent, kill it: %s'
% path)
pghandler.finish()
log.info('Finished with the catalog, removed a total of %d items' %
removed)
log.info("Length: %d, len(uids): %d, len(paths): %d" %
(zcat._length.value, len(zcat.uids), len(zcat.paths)))
transaction.commit()
def manage_reindexIndex(self, ids=None, REQUEST=None, RESPONSE=None,
URL1=None):
"""Reindex indexe(s) from a ZCatalog"""
if not ids:
raise BadRequest('No items specified')
pgthreshold = self._getProgressThreshold()
handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
self.reindexIndex(ids, REQUEST, handler)
if REQUEST and RESPONSE:
RESPONSE.redirect(
URL1 +
'/manage_catalogIndexes'
'?manage_tabs_message=Reindexing%20Performed')
def _update_sortable_title(catalog, change):
update_metadata = 'sortable_title' in catalog.schema
pghandler = ZLogHandler(1000)
logger.info('Updating sortable_title index')
pghandler.init('Updating sortable_title index', len(change))
for i, rid in enumerate(change):
pghandler.report(i)
brain = catalog[rid]
try:
obj = brain.getObject()
except (AttributeError, KeyError):
continue
if update_metadata:
obj.reindexObject()
else:
obj.reindexObject(idxs=['sortable_title'])
pghandler.finish()
logger.info('Updated `sortable_title` index.')
def reindex_mime_type(context):
# Attention, this relies heavily on catalog internals.
# get the more visible zcatalog
zcatalog = getToolByName(context, 'portal_catalog')
# get the more hidden, inner (real) catalog implementation
catalog = zcatalog._catalog
try:
metadata_index = catalog.names.index('mime_type')
except ValueError:
logger.info('`mime_type` not in metadata, skip upgrade step')
return
# we are interested in dexterity and archtetype images and files:
ifaces = ['plone.app.contenttypes.interfaces.IImage',
'plone.app.contenttypes.interfaces.IFile',
'Products.ATContentTypes.interfaces.file.IFileContent',
'Products.ATContentTypes.interfaces.image.IImageContent']
cnt = 0
results = zcatalog.unrestrictedSearchResults(object_provides=ifaces)
num_objects = len(results)
pghandler = ZLogHandler(1000)
pghandler.init('Updating mime_type metadata', num_objects)
for brain in results:
pghandler.report(cnt)
try:
obj = brain._unrestrictedGetObject()
except (AttributeError, KeyError, TypeError, NotFound):
continue
if not obj:
continue
# First get the new value:
new_value = ''
try: # Dexterity
new_value = obj.content_type()
except TypeError: # Archetypes
new_value = obj.content_type
except AttributeError:
continue
# We can now update the record with the new mime_type value
record = list(catalog.data[brain.getRID()])
record[metadata_index] = new_value
catalog.data[brain.getRID()] = tuple(record)
cnt += 1
pghandler.finish()
logger.info('Reindexed `mime_type` for %s items' % str(cnt))
def manage_reindexIndex(self, ids=None, REQUEST=None, RESPONSE=None,
URL1=None):
"""Reindex indexe(s) from a ZCatalog"""
if not ids:
return MessageDialog(title='No items specified',
message='No items were specified!',
action="./manage_catalogIndexes")
pgthreshold = self._getProgressThreshold()
handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
self.reindexIndex(ids, REQUEST, handler)
if REQUEST and RESPONSE:
RESPONSE.redirect(
URL1 +
'/manage_catalogIndexes'
'?manage_tabs_message=Reindexing%20Performed')
def refresh_getIcon_catalog_metadata(context):
"""
get_icon redefined: now boolean:
needs to update metadata
true if item is type image or has image field (named 'image')
e.g. leadimage
see https://github.com/plone/Products.CMFPlone/issues/1226
"""
# Attention, this relies heavily on catalog internals.
# get the more visible zcatalog
zcatalog = getToolByName(context, 'portal_catalog')
# get the more hidden, inner (real) catalog implementation
catalog = zcatalog._catalog
try:
# Get the getIcon index value and
# check if there is a getIcon at all, this may not exist in some
# customizations, who knows, but always exists in default Plone
metadata_index = catalog.names.index('getIcon')
except ValueError:
logger.info('`getIcon` not in metadata, skip upgrade step')
return
cnt = 0
# search whole catalog
results = zcatalog.unrestrictedSearchResults()
num_objects = len(results)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
for brain in results:
pghandler.report(cnt)
# First get the new value
try:
obj = brain._unrestrictedGetObject()
except (AttributeError, KeyError, TypeError, NotFound):
continue
new_value = bool(getattr(aq_base(obj), 'image', False))
# We can now update the record with the new getIcon value
record = list(catalog.data[brain.getRID()])
record[metadata_index] = new_value
catalog.data[brain.getRID()] = tuple(record)
cnt += 1 # we are curious
# done
pghandler.finish()
logger.info('Reindexed `getIcon` for %s items', str(cnt))
def addColumn(self, name, default_value=None, threshold=10000):
"""Adds a row to the meta data schema"""
schema = self.schema
names = list(self.names)
threshold = threshold if threshold is not None else 10000
if name != name.strip():
# Someone could have mistakenly added a space at the end
# of the input field.
LOG.warning('stripped space from new column %r -> %r', name,
name.strip())
name = name.strip()
if name in schema:
raise CatalogError('The column %s already exists' % name)
if name[0] == '_':
raise CatalogError('Cannot cache fields beginning with "_"')
values = schema.values()
if values:
schema[name] = max(values) + 1
else:
schema[name] = 0
names.append(name)
if default_value in (None, ''):
default_value = MV
if len(self):
pghandler = ZLogHandler(threshold)
pghandler.init('Adding %s column' % name, len(self))
for i, (key, value) in enumerate(self.data.iteritems()):
pghandler.report(i)
self.data[key] = value + (default_value, )
pghandler.finish()
self.names = tuple(names)
self.schema = schema
# new column? update the brain
self.updateBrains()
def manage_catalogReindex(self, REQUEST, RESPONSE, URL1):
""" clear the catalog, then re-index everything """
elapse = time.time()
c_elapse = time.clock()
pgthreshold = self._getProgressThreshold()
handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
self.refreshCatalog(clear=1, pghandler=handler)
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
RESPONSE.redirect(URL1 +
'/manage_catalogAdvanced?manage_tabs_message=' +
quote('Catalog Updated \n'
'Total time: %r\n'
'Total CPU time: %r' % (elapse, c_elapse)))
def refreshDatabase(self,
catalogs=True,
catalogsToRebuild=['portal_catalog'],
workflows=False,
workflowsToUpdate=[],
catalogsToUpdate=('portal_catalog', 'reference_catalog', 'uid_catalog')):
"""After the migration script has been executed, it can be necessary to
update the Plone catalogs and/or the workflow settings on every
database object if workflow definitions have changed. We can pass
catalog ids we want to 'clear and rebuild' using
p_catalogsToRebuild."""
if catalogs:
# Manage the catalogs we want to clear and rebuild
# We have to call another method as clear=1 passed to refreshCatalog
# does not seem to work as expected...
for catalogId in catalogsToRebuild:
logger.info('Clearing and rebuilding {0}...'.format(catalogId))
catalogObj = getattr(self.portal, catalogId)
if base_hasattr(catalogObj, 'clearFindAndRebuild'):
catalogObj.clearFindAndRebuild()
else:
# special case for the uid_catalog
catalogObj.manage_rebuildCatalog()
for catalogId in catalogsToUpdate:
if catalogId not in catalogsToRebuild:
logger.info('Refreshing {0}...'.format(catalogId))
catalogObj = getattr(self.portal, catalogId)
pghandler = ZLogHandler()
catalogObj.refreshCatalog(clear=0, pghandler=pghandler)
if workflows:
logger.info('Refresh workflow-related information on every object of the database...')
if not workflowsToUpdate:
logger.info('Refreshing every workflows...')
count = self.wfTool.updateRoleMappings()
else:
wfs = {}
for wf_id in workflowsToUpdate:
logger.info('Refreshing workflow(s) "{0}"...'.format(
", ".join(workflowsToUpdate)))
wf = self.wfTool.getWorkflowById(wf_id)
wfs[wf_id] = wf
count = self.wfTool._recursiveUpdateRoleMappings(self.portal, wfs)
logger.info('{0} object(s) updated.'.format(count))
def reindex_mime_type(context):
# Attention, this relies heavily on catalog internals.
# get the more visible zcatalog
zcatalog = getToolByName(context, 'portal_catalog')
# get the more hidden, inner (real) catalog implementation
catalog = zcatalog._catalog
try:
metadata_index = catalog.names.index('mime_type')
except ValueError:
logger.info('`mime_type` not in metadata, skip upgrade step')
return
# we are interested in dexterity and archtetype images and files:
ifaces = ['plone.app.contenttypes.interfaces.IImage',
'plone.app.contenttypes.interfaces.IFile',
'Products.ATContentTypes.interfaces.file.IFileContent',
'Products.ATContentTypes.interfaces.image.IImageContent']
cnt = 0
results = zcatalog.unrestrictedSearchResults(object_provides=ifaces)
num_objects = len(results)
pghandler = ZLogHandler(1000)
pghandler.init('Updating mime_type metadata', num_objects)
for brain in results:
pghandler.report(cnt)
try:
obj = brain._unrestrictedGetObject()
except (AttributeError, KeyError, TypeError, NotFound):
continue
if not obj:
continue
# First get the new value:
new_value = ''
try: # Dexterity
new_value = obj.content_type()
except TypeError: # Archetypes
new_value = obj.content_type
except AttributeError:
continue
# We can now update the record with the new mime_type value
record = list(catalog.data[brain.getRID()])
record[metadata_index] = new_value
catalog.data[brain.getRID()] = tuple(record)
cnt += 1
pghandler.finish()
logger.info('Reindexed `mime_type` for %s items', str(cnt))
def cleanup(self):
cat = getToolByName(self, "portal_catalog")
purl = getToolByName(self, "portal_url")
portal = purl.getPortalObject()
zcat = cat._catalog
pghandler = ZLogHandler(1000)
size = len(zcat.uids)
pghandler.init('Remove stale items from catalog', size)
cnt = removed = 0
for path in zcat.uids.keys():
pghandler.report(cnt)
cnt += 1
try:
obj = portal.restrictedTraverse(path)
except:
# XXX what to do?
log.warning("no object found for %s" % path)
cat.uncatalog_object(path)
removed += 1
log.info('Kill old path: %s' % path)
else:
if "/".join(obj.getPhysicalPath()) != path:
# saved under an old path! kill it
cat.uncatalog_object(path)
#index = zcat.uids[path]
#del zcat.uids[path]
#del zcat.paths[index]
#zcat._length.change(-1)
removed += 1
log.info('Kill old path: %s' % path)
else:
parent = aq_parent(obj)
ids = [x['id'] for x in parent._objects]
if obj.getId() not in ids and obj.getId() not in parent.objectIds():
cat.uncatalog_object(path)
removed += 1
log.info('ID not found as a child of current parent, kill it: %s' % path)
pghandler.finish()
log.info('Finished with the catalog, removed a total of %d items' % removed)
log.info("Length: %d, len(uids): %d, len(paths): %d" % (zcat._length.value,
len(zcat.uids), len(zcat.paths)))
transaction.commit()
def addColumn(self, name, default_value=None, threshold=10000):
"""Adds a row to the meta data schema"""
schema = self.schema
names = list(self.names)
threshold = threshold if threshold is not None else 10000
if name != name.strip():
# Someone could have mistakenly added a space at the end
# of the input field.
LOG.warning('stripped space from new column %r -> %r', name,
name.strip())
name = name.strip()
if name in schema:
raise CatalogError('The column %s already exists' % name)
if name[0] == '_':
raise CatalogError('Cannot cache fields beginning with "_"')
values = schema.values()
if values:
schema[name] = max(values) + 1
else:
schema[name] = 0
names.append(name)
if default_value in (None, ''):
default_value = MV
if len(self):
pghandler = ZLogHandler(threshold)
pghandler.init('Adding %s column' % name, len(self))
for i, (key, value) in enumerate(self.data.iteritems()):
pghandler.report(i)
self.data[key] = value + (default_value, )
pghandler.finish()
self.names = tuple(names)
self.schema = schema
# new column? update the brain
self.updateBrains()
def fix_themes_navigation(context):
""" Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
objects.
"""
ctool = getToolByName(context, 'portal_catalog')
brains = ctool.unrestrictedSearchResults(
object_provides='eea.themecentre.interfaces.IThemeCentre')
logger.info("Fixing eea.themecentre navigation ...")
pghandler = ZLogHandler(100)
pghandler.init('Fixing eea.themecentre navigation', len(brains))
for index, brain in enumerate(brains):
doc = brain.getObject()
if not INavigationRoot.providedBy(doc):
alsoProvides(doc, INavigationRoot)
pghandler.report(index)
if index % 100 == 0:
transaction.savepoint(optimistic=True)
transaction.savepoint(optimistic=True)
pghandler.finish()
logger.info("Fixing eea.themecentre navigation ... DONE")
def agenda_item_project_set_summary_html(context):
"""Set the mimetype of the summary to text/html.
Otherwise when editing you get a bare textarea and cannot save it.
"""
catalog = getToolByName(context, 'portal_catalog')
brains = catalog(portal_type='AgendaItemProject')
num_objects = len(brains)
logger.info("Updating agenda item summaries.")
fixed = 0
pghandler = ZLogHandler(100)
pghandler.init('Setting mimetype html for agenda item summaries',
num_objects)
for index, brain in enumerate(brains):
pghandler.report(index)
obj = brain.getObject()
if obj.summary.mimetype != 'text/html':
obj.summary.mimetype = 'text/html'
fixed += 1
pghandler.finish()
logger.info("Have set summary mimetype to text/html for {0}/{1} agenda "
"items.".format(fixed, len(brains)))
def update_catalog_metadata(context, types=None):
"""Update catalog metadata.
Adapted from updateIconMetadata in plone.app.upgrade v40/betas.py.
"""
catalog = getToolByName(context, 'portal_catalog')
search = catalog.unrestrictedSearchResults
if types is None:
logger.info('Updating catalog metadata.')
brains = search(sort_on="path")
else:
if isinstance(types, basestring):
types = [types]
logger.info('Updating catalog metadata for %s.', ', '.join(types))
brains = search(portal_type=types, sort_on="path")
num_objects = len(brains)
# Yes, this logs quite often, but I just know I am going to thank
# myself for that... Feedback about progress is good for my heart.
# [Maurits]
pghandler = ZLogHandler(10)
pghandler.init('Updating catalog metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Done updating catalog metadata.')
def fix_themes_navigation(context):
""" Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
objects.
"""
ctool = getToolByName(context, 'portal_catalog')
brains = ctool.unrestrictedSearchResults(
object_provides='eea.themecentre.interfaces.IThemeCentre')
logger.info("Fixing eea.themecentre navigation ...")
pghandler = ZLogHandler(100)
pghandler.init('Fixing eea.themecentre navigation', len(brains))
for index, brain in enumerate(brains):
doc = brain.getObject()
if not INavigationRoot.providedBy(doc):
alsoProvides(doc, INavigationRoot)
pghandler.report(index)
if index % 100 == 0:
transaction.savepoint(optimistic=True)
transaction.savepoint(optimistic=True)
pghandler.finish()
logger.info("Fixing eea.themecentre navigation ... DONE")
def reindex_sortable_title(context):
from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
catalog = getToolByName(context, 'portal_catalog')
_catalog = catalog._catalog
indexes = _catalog.indexes
sort_title_index = indexes.get('sortable_title', None)
if sort_title_index is None:
return
from Products.PluginIndexes.FieldIndex import FieldIndex
if not isinstance(sort_title_index, FieldIndex.FieldIndex):
return
change = []
pghandler = ZLogHandler(10000)
logger.info('Analyzing sortable_title index')
pghandler.init('Analyzing sortable_title index', len(sort_title_index))
for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
pghandler.report(i)
if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
if hasattr(rids, 'keys'):
change.extend(list(rids.keys()))
else:
change.append(rids)
pghandler.finish()
update_metadata = 'sortable_title' in _catalog.schema
pghandler = ZLogHandler(1000)
logger.info('Updating sortable_title index')
pghandler.init('Updating sortable_title index', len(change))
for i, rid in enumerate(change):
pghandler.report(i)
brain = _catalog[rid]
try:
obj = brain.getObject()
except (AttributeError, KeyError):
continue
if update_metadata:
obj.reindexObject()
else:
obj.reindexObject(idxs=['sortable_title'])
pghandler.finish()
logger.info('Updated `sortable_title` index.')
def updateIconMetadata(context):
"""Update getIcon metadata column for all core content"""
catalog = getToolByName(context, 'portal_catalog')
logger.info('Updating `getIcon` metadata.')
search = catalog.unrestrictedSearchResults
_catalog = getattr(catalog, '_catalog', None)
getIconPos = None
if _catalog is not None:
metadata = _catalog.data
getIconPos = _catalog.schema.get('getIcon', None)
typesToUpdate = {
'Document' : ('document_icon.gif', 'document_icon.png'),
'Event' : ('event_icon.gif', 'event_icon.png'),
'File' : ('file_icon.gif', 'file_icon.png'),
'Folder' : ('folder_icon.gif', 'folder_icon.png'),
'Image' : ('image_icon.gif', 'image_icon.png'),
'Link' : ('link_icon.gif', 'link_icon.png'),
'News Item' : ('newsitem_icon.gif', 'newsitem_icon.png'),
'Topic' : ('topic_icon.gif', 'topic_icon.png'),
}
ttool = getToolByName(context, 'portal_types')
empty_icons = []
for name in typesToUpdate.keys():
fti = ttool.get(name)
if fti:
icon_expr = fti.getIconExprObject()
if not icon_expr:
empty_icons.append(name)
brains = search(portal_type=empty_icons, sort_on="path")
num_objects = len(brains)
pghandler = ZLogHandler(1000)
pghandler.init('Updating getIcon metadata', num_objects)
i = 0
for brain in brains:
pghandler.report(i)
brain_icon = brain.getIcon
if not brain_icon:
continue
old_icons = typesToUpdate[brain.portal_type]
if getIconPos is not None:
# if the old icon is a standard icon, we assume no customization
# has taken place and we can simply empty the getIcon metadata
# without loading the object
new_value = ''
if brain_icon not in old_icons:
# Otherwise we need to ask the object
new_value = ''
obj = brain.getObject()
method = getattr(aq_base(obj), 'getIcon', None)
if method is not None:
try:
new_value = obj.getIcon
if callable(new_value):
new_value = new_value()
except ConflictError:
raise
except Exception:
new_value = ''
if brain_icon != new_value:
rid = brain.getRID()
record = metadata[rid]
new_record = list(record)
new_record[getIconPos] = new_value
metadata[rid] = tuple(new_record)
else:
# If we don't have a standard catalog tool, fall back to the
# official API
obj = brain.getObject()
# passing in a valid but inexpensive index, makes sure we don't
# reindex the entire catalog including expensive indexes like
# SearchableText
brain_path = brain.getPath()
try:
catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
except ConflictError:
raise
except Exception:
pass
i += 1
pghandler.finish()
logger.info('Updated `getIcon` metadata.')
