Exemplo n.º 1
0
 def reindexIndexes(self, idxs=[], update_metadata=False, meta_types=[], portal_types=[]):
     """Reindex index including metadata if p_update_metadata=True.
        Filter meta_type/portal_type when p_meta_types and p_portal_types are given."""
     catalog = api.portal.get_tool('portal_catalog')
     paths = catalog._catalog.uids.keys()
     pghandler = ZLogHandler(steps=1000)
     i = 0
     pghandler.info(
         'In reindexIndexes, idxs={0}, update_metadata={1}, meta_types={2}, portal_types={3}'.format(
             repr(idxs), repr(update_metadata), repr(meta_types), repr(portal_types)))
     pghandler.init('reindexIndexes', len(paths))
     for p in paths:
         i += 1
         if pghandler:
             pghandler.report(i)
         obj = catalog.resolve_path(p)
         if obj is None:
             logger.error(
                 'reindexIndex could not resolve an object from the uid %r.' % p)
         else:
             if (meta_types and obj.meta_type not in meta_types) or \
                (portal_types and obj.portal_type not in portal_types):
                 continue
             catalog.catalog_object(
                 obj, p, idxs=idxs, update_metadata=update_metadata, pghandler=pghandler)
     if pghandler:
         pghandler.finish()
def uidDuplications(self):
    pc = getToolByName(self, 'portal_catalog')
    ptool = getToolByName(self, 'portal_url')
    pghandler = ZLogHandler(1000)
    portal = ptool.getPortalObject()
    basecat = pc._catalog
    index = basecat.indexes['UID']
    size = len(index._index.keys())
    pghandler.init('Remove stale items from catalog', size)
    cnt = 0
    for uid, rid in index._index.items():
        path = basecat.paths[rid]
        pghandler.report(cnt)
        cnt += 1
        try:
            obj = portal.restrictedTraverse(path)
        except:
#            print "could not get obj for path %s, skipping" % path
           # we ignore this for the moment
            continue
        parent = aq_parent(obj)
        ids = [x['id'] for x in parent._objects]
        if obj.getId() not in ids and obj.getId() not in parent.objectIds():
            print "path %s not in parent " % path
            del index._index[uid]
            index._length.change(-1)

    return "done"
Exemplo n.º 3
0
    def _migrateMeetingCategoryToDX(self):
        '''Migrate from AT MeetingCategory to DX meetingcategory.'''
        logger.info('Migrating MeetingCategory from AT to DX...')
        # update item classifier
        # migrate references
        pghandler = ZLogHandler(steps=100)
        brains = self.portal.reference_catalog(
            relationship='ItemClassification')
        pghandler.init('Updating field MeetingItem.classifier...', len(brains))
        pghandler.info('Updating field MeetingItem.classifier...')
        i = 0
        for brain in brains:
            i += 1
            pghandler.report(i)
            relation = brain.getObject()
            item = relation.getSourceObject()
            classifier = relation.getTargetObject()
            item.setClassifier(classifier.getId())
            item.reindexObject(idxs=['getRawClassifier'])
        # deleteReferences in a second phase
        for brain in brains:
            relation = brain.getObject()
            item = relation.getSourceObject()
            item.deleteReferences('ItemClassification')
        pghandler.finish()

        logger.info('Migrating categories and classifiers in configuration...')
        # make sure new portal_type meetingcategory is installed
        self.ps.runImportStepFromProfile(
            'profile-Products.PloneMeeting:default', 'typeinfo')
        # make sure no workflow used for meetingcategory
        self.wfTool.setChainForPortalTypes(('meetingcategory', ), ('', ))
        # adapt allowed_types for each MeetingConfig.categories/classifiers folders
        for cfg in self.tool.objectValues('MeetingConfig'):
            for folder_id in ('categories', 'classifiers'):
                constrain = IConstrainTypes(getattr(cfg, folder_id))
                constrain.setConstrainTypesMode(1)
                allowedTypes = ['meetingcategory']
                constrain.setLocallyAllowedTypes(allowedTypes)
                constrain.setImmediatelyAddableTypes(allowedTypes)
        # migrate to DX
        pac_migrate(self.portal, MeetingCategoryMigrator)
        self.removeUnusedPortalTypes(portal_types=['MeetingCategory'])
        # add meetingcategory to types_not_searched
        props = api.portal.get_tool('portal_properties').site_properties
        nsTypes = props.getProperty('types_not_searched')
        if 'meetingcategory' not in nsTypes:
            nsTypes = list(nsTypes)
            # MeetingCategory was removed by removeUnusedPortalTypes
            nsTypes.append('meetingcategory')
            props.manage_changeProperties(types_not_searched=tuple(nsTypes))
        logger.info('Done.')
Exemplo n.º 4
0
    def delColumn(self, name, threshold=10000):
        """Deletes a row from the meta data schema"""
        names = list(self.names)
        _index = names.index(name)
        threshold = threshold if threshold is not None else 10000

        if name not in self.schema:
            LOG.error(
                'delColumn attempted to delete nonexistent '
                'column %s.', str(name))
            return

        del names[_index]

        # rebuild the schema
        schema = {}
        for i, name in enumerate(names):
            schema[name] = i

        self.schema = schema
        self.names = tuple(names)

        # update the brain
        self.updateBrains()

        # remove the column value from each record
        if len(self):
            _next_index = _index + 1
            pghandler = ZLogHandler(threshold)
            pghandler.init('Deleting %s column' % name, len(self))
            for i, (key, value) in enumerate(self.data.iteritems()):
                pghandler.report(i)
                self.data[key] = value[:_index] + value[_next_index:]
            pghandler.finish()
Exemplo n.º 5
0
 def _cleanFTWLabels(self):
     """This fix partial migrations of stored ftw.labels:labeling that are
        still PersistentList and not PersistentMapping."""
     logger.info("Cleaning ftw.labels wrong annotations...")
     brains = self.catalog(object_provides=ILabelSupport.__identifier__)
     pghandler = ZLogHandler(steps=100)
     pghandler.init('Cleaning ftw.labels wrong annotations...', len(brains))
     i = 0
     cleaned = 0
     for brain in brains:
         i += 1
         pghandler.report(i)
         obj = brain.getObject()
         annotations = IAnnotations(obj)
         if 'ftw.labels:labeling' in annotations and \
            not isinstance(annotations['ftw.labels:labeling'], PersistentMapping):
             if annotations['ftw.labels:labeling']:
                 labeling = ILabeling(obj)
                 old_values = [label for label in labeling.storage]
                 del annotations['ftw.labels:labeling']
                 labeling._storage = None
                 labeling.update(old_values)
                 logger.info('In _cleanFTWLabels, cleaned %s' %
                             brain.getPath())
             else:
                 del annotations['ftw.labels:labeling']
             obj.reindexObject(idxs=['labels'])
             cleaned += 1
     if cleaned:
         self.warn(logger,
                   'In _cleanFTWLabels, cleaned %s element(s)' % cleaned)
     pghandler.finish()
     logger.info('Done.')
Exemplo n.º 6
0
def reindex_sortable_title(context):
    from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
    catalog = getToolByName(context, 'portal_catalog')
    _catalog = catalog._catalog
    indexes = _catalog.indexes
    sort_title_index = indexes.get('sortable_title', None)
    if sort_title_index is None:
        return
    from Products.PluginIndexes.FieldIndex import FieldIndex
    if not isinstance(sort_title_index, FieldIndex.FieldIndex):
        return
    change = []
    pghandler = ZLogHandler(10000)
    logger.info('Analyzing sortable_title index')
    pghandler.init('Analyzing sortable_title index', len(sort_title_index))
    for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
        pghandler.report(i)
        if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
            try:
                keys = rids.keys()
            except AttributeError:
                change.append(rids)
            else:
                change.extend(list(keys))
    pghandler.finish()
    # flake8 complained that reindex_sortable_title is too complex (11).
    # So we split it in two.
    _update_sortable_title(_catalog, change)
Exemplo n.º 7
0
    def _migrateLabelForCouncil(self):
        """Field labelForCouncil is replaced by
           otherMeetingConfigsClonableToFieldDetailedDescription in College and
           detailedDescription in Council."""
        logger.info(
            'Migrating field "labelForCouncil" in "meeting-config-college"...')
        # enable relevant fields in MeetingConfigs
        # College we use the "otherMeetingConfigsClonableToFieldLabelForCouncil" field
        # Council is correct
        college_cfg = self.tool.get('meeting-config-college')
        used_attrs = college_cfg.getUsedItemAttributes()
        used_attrs = replace_in_list(
            used_attrs, "labelForCouncil",
            "otherMeetingConfigsClonableToFieldLabelForCouncil")
        college_cfg.setUsedItemAttributes(used_attrs)
        logger.info('Done.')

        logger.info('Migrating field "labelForCouncil" for College items...')
        pghandler = ZLogHandler(steps=1000)
        brains = self.catalog(portal_type='MeetingItemCollege')
        pghandler.init('Migrating field "labelForCouncil" for College items',
                       len(brains))
        i = 0
        for brain in brains:
            i += 1
            pghandler.report(i)
            item = brain.getObject()
            if not item.fieldIsEmpty('labelForCouncil'):
                item.setOtherMeetingConfigsClonableToFieldLabelForCouncil(
                    item.getRawLabelForCouncil())
                item.setLabelForCouncil('')
        pghandler.finish()
        logger.info('Done.')
Exemplo n.º 8
0
 def _umarkCreationFlagForEveryItems(self):
     """ """
     brains = self.catalog(meta_type='MeetingItem')
     pghandler = ZLogHandler(steps=100)
     pghandler.init('Cleaning ftw.labels wrong annotations...', len(brains))
     pghandler.info("Unmarking _at_creation_flag for every items...")
     i = 0
     for brain in brains:
         i += 1
         pghandler.report(i)
         item = brain.getObject()
         item.unmarkCreationFlag()
     pghandler.finish()
     logger.info('Done.')
Exemplo n.º 9
0
 def import_localroles(self, data):
     results = 0
     total = len(data)
     for index, item in enumerate(data, start=1):
         obj = api.content.get(UID=item["uuid"])
         if not obj:
             continue
         if item.get("localroles"):
             localroles = item["localroles"]
             for userid in localroles:
                 obj.manage_setLocalRoles(userid=userid,
                                          roles=localroles[userid])
             logger.debug(u"Set roles on {}: {}".format(
                 obj.absolute_url(), localroles))
         if item.get("block"):
             obj.__ac_local_roles_block__ = 1
             logger.debug(u"Diable acquisition of local roles on {}".format(
                 obj.absolute_url()))
         if not index % 1000:
             logger.info(u"Set local roles on {} ({}%) of {} items".format(
                 index, round(index / total * 100, 2), total))
         results += 1
     if results:
         logger.info("Reindexing Security")
         catalog = api.portal.get_tool("portal_catalog")
         pghandler = ZLogHandler(1000)
         catalog.reindexIndex("allowedRolesAndUsers",
                              None,
                              pghandler=pghandler)
     return results
Exemplo n.º 10
0
    def refreshCatalogs(self):
        """
        It reindexes the modified catalogs but, while cleanAndRebuildCatalogs
        recatalogs all objects in the database, this method only reindexes over
        the already cataloged objects.

        If a metacolumn is added it refreshes the catalog, if only a new index
        is added, it reindexes only those new indexes.
        """
        to_refresh = self.refreshcatalog[:]
        to_reindex = self.reindexcatalog.keys()
        to_reindex = to_reindex[:]
        done = []
        # Start reindexing the catalogs with new columns
        for catalog_to_refresh in to_refresh:
            try:
                logger.info('Catalog {0} refreshing started'.format(
                    catalog_to_refresh))
                catalog = getToolByName(self.portal, catalog_to_refresh)
                handler = ZLogHandler(self.pgthreshold)
                catalog.refreshCatalog(pghandler=handler)
                logger.info('Catalog {0} refreshed'.format(catalog_to_refresh))
                transaction.commit()
            except:
                logger.error(
                    'Unable to refresh catalog {0}'.format(catalog_to_refresh))
                raise
            done.append(catalog_to_refresh)
        # Now the catalogs which only need reindxing
        for catalog_to_reindex in to_reindex:
            if catalog_to_reindex in done:
                continue
            try:
                logger.info('Catalog {0} reindexing started'.format(
                    catalog_to_reindex))
                catalog = getToolByName(self.portal, catalog_to_reindex)
                indexes = self.reindexcatalog[catalog_to_reindex]
                handler = ZLogHandler(self.pgthreshold)
                catalog.reindexIndex(indexes, None, pghandler=handler)
                logger.info('Catalog {0} reindexed'.format(catalog_to_reindex))
                transaction.commit()
            except:
                logger.error(
                    'Unable to reindex catalog {0}'.format(catalog_to_reindex))
                raise
            done.append(catalog_to_reindex)
Exemplo n.º 11
0
def update_getIcon(context):
    typesToUpdate = {
        'Location': ('link_icon.gif', 'link_icon.png'),
        'Name': ('document_icon.gif', 'document_icon.png'),
        'PleiadesVocabulary': ('folder_icon.gif', 'folder_icon.png'),
        'PleiadesVocabularyTerm': ('document_icon.gif', 'document_icon.png'),
    }

    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon')

    brains = search(portal_type=typesToUpdate.keys(), sort_on="path")
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        old_icon, new_icon = typesToUpdate[brain.portal_type]
        if brain_icon != new_icon:
            rid = brain.getRID()
            record = metadata[rid]
            new_record = list(record)
            new_record[getIconPos] = new_icon
            metadata[rid] = tuple(new_record)
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')
Exemplo n.º 12
0
    def delColumn(self, name, threshold=10000):
        """Deletes a row from the meta data schema"""
        names = list(self.names)
        _index = names.index(name)
        threshold = threshold if threshold is not None else 10000

        if name not in self.schema:
            LOG.error('delColumn attempted to delete nonexistent '
                      'column %s.', str(name))
            return

        del names[_index]

        # rebuild the schema
        schema = {}
        for i, name in enumerate(names):
            schema[name] = i

        self.schema = schema
        self.names = tuple(names)

        # update the brain
        self.updateBrains()

        # remove the column value from each record
        if len(self):
            _next_index = _index + 1
            pghandler = ZLogHandler(threshold)
            pghandler.init('Deleting %s column' % name, len(self))
            for i, (key, value) in enumerate(self.data.iteritems()):
                pghandler.report(i)
                self.data[key] = value[:_index] + value[_next_index:]
            pghandler.finish()
Exemplo n.º 13
0
def updateIconsInBrains(context, typesToUpdate=None):
    """Update getIcon metadata column in given types.

    typesToUpdate must be a dictionary, for example: {
        # portal_type: ('old_icon.gif', 'new_icon.png'),
        'Document': ('document_icon.gif', 'document_icon.png'),
        }

    The portal_types must have an empty icon_expr, because that is the
    main use case.
    """
    if not typesToUpdate:
        logger.warn('No typesToUpdate given for updateIconsInBrains.')
        return

    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon', None)
    empty_icons = _types_with_empty_icons(context, typesToUpdate)
    brains = search(portal_type=empty_icons, sort_on='path')
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        if not brain_icon:
            continue
        if getIconPos is not None:
            _update_icon_in_single_brain(brain, typesToUpdate, getIconPos,
                                         metadata)
        else:
            # If we don't have a standard catalog tool, fall back to the
            # official API
            obj = brain.getObject()
            # passing in a valid but inexpensive index, makes sure we don't
            # reindex the entire catalog including expensive indexes like
            # SearchableText
            brain_path = brain.getPath()
            try:
                catalog.catalog_object(obj, brain_path, ['id'], True,
                                       pghandler)
            except ConflictError:
                raise
            except Exception:
                pass
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')
Exemplo n.º 14
0
def reindex_getWorksheetUID(portal):
    """Reindex getWorksheetUID index from analysis_catalog
    """
    catalog_name = CATALOG_ANALYSIS_LISTING
    logger.info("Reindexing getWorksheetUID for {} ...".format(catalog_name))
    handler = ZLogHandler(steps=100)
    catalog = api.get_tool(catalog_name)
    catalog.reindexIndex("getWorksheetUID", None, pghandler=handler)
    commit_transaction(portal)
Exemplo n.º 15
0
    def _migrate_categories_to_groups_in_charge_field(self):
        logger.info("Migrating Category to Groups in charge field")
        catalog = api.portal.get_tool("portal_catalog")
        own_org = get_own_organization()

        # First we create the appropriate groupsInCharge
        logger.info("Adapting MeetingConfig...")
        cfg = catalog(portal_type="MeetingConfig",
                      id=self.meeting_config_id)[0].getObject()
        categories = cfg.categories
        for cat_id, category in categories.contentItems():
            if cat_id not in own_org.objectIds():
                data = {
                    "id": cat_id,
                    "title": category.Title(),
                    "description": category.Description()
                }
                api.content.create(container=own_org,
                                   type="organization",
                                   **data)
            new_org_uid = org_id_to_uid(cat_id)

            enabled_plonegroup_org_uids = api.portal.get_registry_record(
                ORGANIZATIONS_REGISTRY)
            if new_org_uid not in enabled_plonegroup_org_uids:
                api.portal.set_registry_record(
                    ORGANIZATIONS_REGISTRY,
                    enabled_plonegroup_org_uids + [new_org_uid])

            ordered_group_in_charge_uids = cfg.getOrderedGroupsInCharge()
            if new_org_uid not in ordered_group_in_charge_uids:
                cfg.setOrderedGroupsInCharge(ordered_group_in_charge_uids +
                                             (new_org_uid, ))

            category.enabled = False

        # Next we migrate the category to groupsInCharge on MeetingItems
        brains = catalog(portal_type=(
            cfg.getItemTypeName(),
            cfg.getItemTypeName(configType="MeetingItemRecurring"),
            cfg.getItemTypeName(configType="MeetingItemTemplate"),
        ))
        pghandler = ZLogHandler(steps=1000)
        pghandler.init("Adapting items...", len(brains))
        for i, brain in enumerate(brains):
            pghandler.report(i)
            item = brain.getObject()
            if item.getCategory() != u"_none_":
                item.setGroupsInCharge([org_id_to_uid(item.getCategory())])
                item.setCategory(None)
                item.reindexObject(idxs=["getGroupsInCharge", "getCategory"])
        pghandler.finish()

        logger.info("Done migrating Categories to Groups in charge field")
Exemplo n.º 16
0
 def rebuild_uid_index(self):
     """Rebuild uid index.
     """
     portal_catalog = getToolByName(self.context, 'portal_catalog')
     portal_catalog._catalog.clearIndex('UID')
     portal_catalog._catalog.reindexIndex('UID',
                                          self.request,
                                          pghandler=ZLogHandler())
     IStatusMessage(self.request).addStatusMessage('Rebuilt UID index.',
                                                   type='info')
     return self.redirect_to_view()
Exemplo n.º 17
0
 def reindexIndexesFor(self, idxs=[], **query):
     """ Reindex p_idxs on objects of given p_portal_types. """
     catalog = api.portal.get_tool('portal_catalog')
     brains = catalog(**query)
     pghandler = ZLogHandler(steps=1000)
     len_brains = len(brains)
     pghandler.info(
         'In reindexIndexesFor, reindexing indexes "{0}" on "{1}" objects ({2})...'.format(
             ', '.join(idxs) or '*',
             len(brains),
             str(query)))
     pghandler.init('reindexIndexesFor', len_brains)
     i = 0
     for brain in brains:
         i += 1
         pghandler.report(i)
         obj = brain.getObject()
         obj.reindexObject(idxs=idxs)
     pghandler.finish()
     logger.info('Done.')
def uidDuplications(self):
    pc = getToolByName(self, 'portal_catalog')
    ptool = getToolByName(self, 'portal_url')
    pghandler = ZLogHandler(1000)
    portal = ptool.getPortalObject()
    basecat = pc._catalog
    index = basecat.indexes['UID']
    size = len(index._index.keys())
    pghandler.init('Remove stale items from catalog', size)
    cnt = 0
    for uid, rid in index._index.items():
        path = basecat.paths[rid]
        pghandler.report(cnt)
        cnt += 1
        try:
            obj = portal.restrictedTraverse(path)
        except:
            #            print "could not get obj for path %s, skipping" % path
            # we ignore this for the moment
            continue
        parent = aq_parent(obj)
        ids = [x['id'] for x in parent._objects]
        if obj.getId() not in ids and obj.getId() not in parent.objectIds():
            print "path %s not in parent " % path
            del index._index[uid]
            index._length.change(-1)

    return "done"
Exemplo n.º 19
0
def updateIconsInBrains(context, typesToUpdate=None):
    """Update getIcon metadata column in given types.

    typesToUpdate must be a dictionary, for example: {
        # portal_type: ('old_icon.gif', 'new_icon.png'),
        'Document': ('document_icon.gif', 'document_icon.png'),
        }

    The portal_types must have an empty icon_expr, because that is the
    main use case.
    """
    if not typesToUpdate:
        logger.warn('No typesToUpdate given for updateIconsInBrains.')
        return

    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon', None)
    empty_icons = _types_with_empty_icons(context, typesToUpdate)
    brains = search(portal_type=empty_icons, sort_on='path')
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        if not brain_icon:
            continue
        if getIconPos is not None:
            _update_icon_in_single_brain(
                brain, typesToUpdate, getIconPos, metadata)
        else:
            # If we don't have a standard catalog tool, fall back to the
            # official API
            obj = brain.getObject()
            # passing in a valid but inexpensive index, makes sure we don't
            # reindex the entire catalog including expensive indexes like
            # SearchableText
            brain_path = brain.getPath()
            try:
                catalog.catalog_object(
                    obj, brain_path, ['id'], True, pghandler)
            except ConflictError:
                raise
            except Exception:
                pass
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')
Exemplo n.º 20
0
    def catalog_rebuild_index(self, name):
        """Reindex the ``portal_catalog`` index identified by ``name``.
        """
        catalog = self.getToolByName('portal_catalog')
        LOG.info("Reindexing index %s" % name)

        # pylint: disable=W0212
        pgthreshold = catalog._getProgressThreshold() or 100
        # pylint: enable=W0212
        pghandler = ZLogHandler(pgthreshold)
        catalog.reindexIndex(name, None, pghandler=pghandler)

        LOG.info("Reindexing index %s DONE" % name)
Exemplo n.º 21
0
def add_metadata(portal, catalog_id, column, refresh_catalog=False):
    logger.info("Adding '{}' metadata to '{}' ...".format(column, catalog_id))
    catalog = api.get_tool(catalog_id)
    if column in catalog.schema():
        logger.info("Metadata '{}' already in catalog '{}' [SKIP]"
                    .format(column, catalog_id))
        return
    catalog.addColumn(column)

    if refresh_catalog:
        logger.info("Refreshing catalog '{}' ...".format(catalog_id))
        handler = ZLogHandler(steps=100)
        catalog.refreshCatalog(pghandler=handler)
Exemplo n.º 22
0
 def _removeMeetingItemGroupedItemsNumAttribute(self):
     """This field was supposed to be used but will not...
        Remove the MeetingItem.groupedItemsNum attribute on every existing items."""
     logger.info('Removing attribute "groupedItemsNum" from every items...')
     brains = self.catalog(meta_type="MeetingItem")
     pghandler = ZLogHandler(steps=1000)
     pghandler.init('Working', len(brains))
     i = 0
     for brain in brains:
         i += 1
         pghandler.report(i)
         item = brain.getObject()
         safe_delattr(item, "groupedItemsNum")
     pghandler.finish()
     logger.info('Done.')
Exemplo n.º 23
0
    def refresh_getIcon_catalog_metadata(context):
        """
        get_icon redefined: now boolean:
        needs to update metadata
        true if item is type image or has image field (named 'image')
        e.g. leadimage
        see https://github.com/plone/Products.CMFPlone/issues/1226
        """
        # Attention, this relies heavily on catalog internals.

        # get the more visible zcatalog
        zcatalog = getToolByName(context, 'portal_catalog')

        # get the more hidden, inner (real) catalog implementation
        catalog = zcatalog._catalog
        try:
            # Get the getIcon index value and
            # check if there is a getIcon at all, this may not exist in some
            # customizations, who knows, but always exists in default Plone
            metadata_index = catalog.names.index('getIcon')
        except ValueError:
            logger.info('`getIcon` not in metadata, skip upgrade step')
            return

        cnt = 0
        # search whole catalog
        results = zcatalog.unrestrictedSearchResults()
        num_objects = len(results)
        pghandler = ZLogHandler(1000)
        pghandler.init('Updating getIcon metadata', num_objects)
        for brain in results:
            pghandler.report(cnt)
            # First get the new value
            try:
                obj = brain._unrestrictedGetObject()
            except (AttributeError, KeyError, TypeError, NotFound):
                continue
            new_value = bool(getattr(aq_base(obj), 'image', False))

            # We can now update the record with the new getIcon value
            record = list(catalog.data[brain.getRID()])
            record[metadata_index] = new_value
            catalog.data[brain.getRID()] = tuple(record)

            cnt += 1  # we are curious
        # done
        pghandler.finish()
        logger.info('Reindexed `getIcon` for %s items' % str(cnt))
Exemplo n.º 24
0
def del_metadata(portal, catalog_id, column, refresh_catalog=False):
    logger.info("Removing '{}' metadata from '{}' ..."
                .format(column, catalog_id))
    catalog = api.get_tool(catalog_id)
    if column not in catalog.schema():
        logger.info("Metadata '{}' not in catalog '{}' [SKIP]"
                    .format(column, catalog_id))
        return
    catalog.delColumn(column)

    if refresh_catalog:
        logger.info("Refreshing catalog '{}' ...".format(catalog_id))
        handler = ZLogHandler(steps=100)
        catalog.refreshCatalog(pghandler=handler)
Exemplo n.º 25
0
def reindex_sortable_title(portal):
    """Reindex sortable_title from some catalogs
    """
    catalogs = [
        "bika_catalog",
        "bika_setup_catalog",
        "portal_catalog",
    ]
    for catalog_name in catalogs:
        logger.info("Reindexing sortable_title for {} ...".format(catalog_name))
        handler = ZLogHandler(steps=100)
        catalog = api.get_tool(catalog_name)
        catalog.reindexIndex("sortable_title", None, pghandler=handler)
        commit_transaction(portal)
def cleanup(self):
    cat = getToolByName(self, "portal_catalog")
    purl = getToolByName(self, "portal_url")
    portal = purl.getPortalObject()
    zcat = cat._catalog
    pghandler = ZLogHandler(1000)
    size = len(zcat.uids)
    pghandler.init('Remove stale items from catalog', size)
    cnt = removed = 0
    for path in zcat.uids.keys():
        pghandler.report(cnt)
        cnt += 1
        try:
            obj = portal.restrictedTraverse(path)
        except:
            # XXX what to do?
            log.warning("no object found for %s" % path)
            cat.uncatalog_object(path)
            removed += 1
            log.info('Kill old path: %s' % path)
        else:
            if "/".join(obj.getPhysicalPath()) != path:
                # saved under an old path! kill it
                cat.uncatalog_object(path)
                #index = zcat.uids[path]
                #del zcat.uids[path]
                #del zcat.paths[index]
                #zcat._length.change(-1)
                removed += 1
                log.info('Kill old path: %s' % path)
            else:
                parent = aq_parent(obj)
                ids = [x['id'] for x in parent._objects]
                if obj.getId() not in ids and obj.getId(
                ) not in parent.objectIds():
                    cat.uncatalog_object(path)
                    removed += 1
                    log.info(
                        'ID not found as a child of current parent, kill it: %s'
                        % path)
    pghandler.finish()
    log.info('Finished with the catalog, removed a total of %d items' %
             removed)
    log.info("Length: %d, len(uids): %d, len(paths): %d" %
             (zcat._length.value, len(zcat.uids), len(zcat.paths)))

    transaction.commit()
Exemplo n.º 27
0
    def manage_reindexIndex(self, ids=None, REQUEST=None, RESPONSE=None,
                            URL1=None):
        """Reindex indexe(s) from a ZCatalog"""
        if not ids:
            raise BadRequest('No items specified')

        pgthreshold = self._getProgressThreshold()
        handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
        self.reindexIndex(ids, REQUEST, handler)

        if REQUEST and RESPONSE:
            RESPONSE.redirect(
                URL1 +
                '/manage_catalogIndexes'
                '?manage_tabs_message=Reindexing%20Performed')
Exemplo n.º 28
0
def _update_sortable_title(catalog, change):
    update_metadata = 'sortable_title' in catalog.schema
    pghandler = ZLogHandler(1000)
    logger.info('Updating sortable_title index')
    pghandler.init('Updating sortable_title index', len(change))
    for i, rid in enumerate(change):
        pghandler.report(i)
        brain = catalog[rid]
        try:
            obj = brain.getObject()
        except (AttributeError, KeyError):
            continue
        if update_metadata:
            obj.reindexObject()
        else:
            obj.reindexObject(idxs=['sortable_title'])
    pghandler.finish()
    logger.info('Updated `sortable_title` index.')
Exemplo n.º 29
0
def reindex_mime_type(context):
    # Attention, this relies heavily on catalog internals.
    # get the more visible zcatalog
    zcatalog = getToolByName(context, 'portal_catalog')
    # get the more hidden, inner (real) catalog implementation
    catalog = zcatalog._catalog
    try:
        metadata_index = catalog.names.index('mime_type')
    except ValueError:
        logger.info('`mime_type` not in metadata, skip upgrade step')
        return
    # we are interested in dexterity and archtetype images and files:
    ifaces = ['plone.app.contenttypes.interfaces.IImage',
              'plone.app.contenttypes.interfaces.IFile',
              'Products.ATContentTypes.interfaces.file.IFileContent',
              'Products.ATContentTypes.interfaces.image.IImageContent']
    cnt = 0
    results = zcatalog.unrestrictedSearchResults(object_provides=ifaces)
    num_objects = len(results)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating mime_type metadata', num_objects)
    for brain in results:
        pghandler.report(cnt)
        try:
            obj = brain._unrestrictedGetObject()
        except (AttributeError, KeyError, TypeError, NotFound):
            continue
        if not obj:
            continue
        # First get the new value:
        new_value = ''
        try:  # Dexterity
            new_value = obj.content_type()
        except TypeError:  # Archetypes
            new_value = obj.content_type
        except AttributeError:
            continue
        # We can now update the record with the new mime_type value
        record = list(catalog.data[brain.getRID()])
        record[metadata_index] = new_value
        catalog.data[brain.getRID()] = tuple(record)
        cnt += 1
    pghandler.finish()
    logger.info('Reindexed `mime_type` for %s items' % str(cnt))
Exemplo n.º 30
0
    def manage_reindexIndex(self, ids=None, REQUEST=None, RESPONSE=None,
                            URL1=None):
        """Reindex indexe(s) from a ZCatalog"""
        if not ids:
            return MessageDialog(title='No items specified',
                message='No items were specified!',
                action="./manage_catalogIndexes")

        pgthreshold = self._getProgressThreshold()
        handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
        self.reindexIndex(ids, REQUEST, handler)

        if REQUEST and RESPONSE:
            RESPONSE.redirect(
                URL1 +
                '/manage_catalogIndexes'
                '?manage_tabs_message=Reindexing%20Performed')
Exemplo n.º 31
0
    def refresh_getIcon_catalog_metadata(context):
        """
        get_icon redefined: now boolean:
        needs to update metadata
        true if item is type image or has image field (named 'image')
        e.g. leadimage
        see https://github.com/plone/Products.CMFPlone/issues/1226
        """
        # Attention, this relies heavily on catalog internals.

        # get the more visible zcatalog
        zcatalog = getToolByName(context, 'portal_catalog')

        # get the more hidden, inner (real) catalog implementation
        catalog = zcatalog._catalog
        try:
            # Get the getIcon index value and
            # check if there is a getIcon at all, this may not exist in some
            # customizations, who knows, but always exists in default Plone
            metadata_index = catalog.names.index('getIcon')
        except ValueError:
            logger.info('`getIcon` not in metadata, skip upgrade step')
            return

        cnt = 0
        # search whole catalog
        results = zcatalog.unrestrictedSearchResults()
        num_objects = len(results)
        pghandler = ZLogHandler(1000)
        pghandler.init('Updating getIcon metadata', num_objects)
        for brain in results:
            pghandler.report(cnt)
            # First get the new value
            try:
                obj = brain._unrestrictedGetObject()
            except (AttributeError, KeyError, TypeError, NotFound):
                continue
            new_value = bool(getattr(aq_base(obj), 'image', False))

            # We can now update the record with the new getIcon value
            record = list(catalog.data[brain.getRID()])
            record[metadata_index] = new_value
            catalog.data[brain.getRID()] = tuple(record)

            cnt += 1  # we are curious
        # done
        pghandler.finish()
        logger.info('Reindexed `getIcon` for %s items', str(cnt))
Exemplo n.º 32
0
    def addColumn(self, name, default_value=None, threshold=10000):
        """Adds a row to the meta data schema"""
        schema = self.schema
        names = list(self.names)
        threshold = threshold if threshold is not None else 10000

        if name != name.strip():
            # Someone could have mistakenly added a space at the end
            # of the input field.
            LOG.warning('stripped space from new column %r -> %r', name,
                        name.strip())
            name = name.strip()

        if name in schema:
            raise CatalogError('The column %s already exists' % name)

        if name[0] == '_':
            raise CatalogError('Cannot cache fields beginning with "_"')

        values = schema.values()
        if values:
            schema[name] = max(values) + 1
        else:
            schema[name] = 0
        names.append(name)

        if default_value in (None, ''):
            default_value = MV

        if len(self):
            pghandler = ZLogHandler(threshold)
            pghandler.init('Adding %s column' % name, len(self))
            for i, (key, value) in enumerate(self.data.iteritems()):
                pghandler.report(i)
                self.data[key] = value + (default_value, )
            pghandler.finish()

        self.names = tuple(names)
        self.schema = schema

        # new column? update the brain
        self.updateBrains()
Exemplo n.º 33
0
    def manage_catalogReindex(self, REQUEST, RESPONSE, URL1):
        """ clear the catalog, then re-index everything """

        elapse = time.time()
        c_elapse = time.clock()

        pgthreshold = self._getProgressThreshold()
        handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
        self.refreshCatalog(clear=1, pghandler=handler)

        elapse = time.time() - elapse
        c_elapse = time.clock() - c_elapse

        RESPONSE.redirect(URL1 +
                          '/manage_catalogAdvanced?manage_tabs_message=' +
                          quote('Catalog Updated \n'
                                'Total time: %r\n'
                                'Total CPU time: %r' % (elapse, c_elapse)))
Exemplo n.º 34
0
 def refreshDatabase(self,
                     catalogs=True,
                     catalogsToRebuild=['portal_catalog'],
                     workflows=False,
                     workflowsToUpdate=[],
                     catalogsToUpdate=('portal_catalog', 'reference_catalog', 'uid_catalog')):
     """After the migration script has been executed, it can be necessary to
        update the Plone catalogs and/or the workflow settings on every
        database object if workflow definitions have changed. We can pass
        catalog ids we want to 'clear and rebuild' using
        p_catalogsToRebuild."""
     if catalogs:
         # Manage the catalogs we want to clear and rebuild
         # We have to call another method as clear=1 passed to refreshCatalog
         # does not seem to work as expected...
         for catalogId in catalogsToRebuild:
             logger.info('Clearing and rebuilding {0}...'.format(catalogId))
             catalogObj = getattr(self.portal, catalogId)
             if base_hasattr(catalogObj, 'clearFindAndRebuild'):
                 catalogObj.clearFindAndRebuild()
             else:
                 # special case for the uid_catalog
                 catalogObj.manage_rebuildCatalog()
         for catalogId in catalogsToUpdate:
             if catalogId not in catalogsToRebuild:
                 logger.info('Refreshing {0}...'.format(catalogId))
                 catalogObj = getattr(self.portal, catalogId)
                 pghandler = ZLogHandler()
                 catalogObj.refreshCatalog(clear=0, pghandler=pghandler)
     if workflows:
         logger.info('Refresh workflow-related information on every object of the database...')
         if not workflowsToUpdate:
             logger.info('Refreshing every workflows...')
             count = self.wfTool.updateRoleMappings()
         else:
             wfs = {}
             for wf_id in workflowsToUpdate:
                 logger.info('Refreshing workflow(s) "{0}"...'.format(
                     ", ".join(workflowsToUpdate)))
                 wf = self.wfTool.getWorkflowById(wf_id)
                 wfs[wf_id] = wf
             count = self.wfTool._recursiveUpdateRoleMappings(self.portal, wfs)
         logger.info('{0} object(s) updated.'.format(count))
Exemplo n.º 35
0
def reindex_mime_type(context):
    # Attention, this relies heavily on catalog internals.
    # get the more visible zcatalog
    zcatalog = getToolByName(context, 'portal_catalog')
    # get the more hidden, inner (real) catalog implementation
    catalog = zcatalog._catalog
    try:
        metadata_index = catalog.names.index('mime_type')
    except ValueError:
        logger.info('`mime_type` not in metadata, skip upgrade step')
        return
    # we are interested in dexterity and archtetype images and files:
    ifaces = ['plone.app.contenttypes.interfaces.IImage',
              'plone.app.contenttypes.interfaces.IFile',
              'Products.ATContentTypes.interfaces.file.IFileContent',
              'Products.ATContentTypes.interfaces.image.IImageContent']
    cnt = 0
    results = zcatalog.unrestrictedSearchResults(object_provides=ifaces)
    num_objects = len(results)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating mime_type metadata', num_objects)
    for brain in results:
        pghandler.report(cnt)
        try:
            obj = brain._unrestrictedGetObject()
        except (AttributeError, KeyError, TypeError, NotFound):
            continue
        if not obj:
            continue
        # First get the new value:
        new_value = ''
        try:  # Dexterity
            new_value = obj.content_type()
        except TypeError:  # Archetypes
            new_value = obj.content_type
        except AttributeError:
            continue
        # We can now update the record with the new mime_type value
        record = list(catalog.data[brain.getRID()])
        record[metadata_index] = new_value
        catalog.data[brain.getRID()] = tuple(record)
        cnt += 1
    pghandler.finish()
    logger.info('Reindexed `mime_type` for %s items', str(cnt))
def cleanup(self):
    cat = getToolByName(self, "portal_catalog")
    purl = getToolByName(self, "portal_url")
    portal = purl.getPortalObject()
    zcat = cat._catalog
    pghandler = ZLogHandler(1000)
    size = len(zcat.uids)
    pghandler.init('Remove stale items from catalog', size)
    cnt = removed = 0
    for path in zcat.uids.keys():
        pghandler.report(cnt)
        cnt += 1
        try:
            obj = portal.restrictedTraverse(path)
        except:
            # XXX what to do?
            log.warning("no object found for %s" % path)
            cat.uncatalog_object(path)
            removed += 1
            log.info('Kill old path: %s' % path)
        else:
            if  "/".join(obj.getPhysicalPath()) != path:
                # saved under an old path! kill it
                cat.uncatalog_object(path)
                #index = zcat.uids[path]
                #del zcat.uids[path]
                #del zcat.paths[index]
                #zcat._length.change(-1)
                removed += 1
                log.info('Kill old path: %s' % path)
            else:
                parent = aq_parent(obj)
                ids = [x['id'] for x in parent._objects]
                if obj.getId() not in ids and obj.getId() not in parent.objectIds():
                    cat.uncatalog_object(path)
                    removed += 1
                    log.info('ID not found as a child of current parent, kill it: %s' % path)
    pghandler.finish()
    log.info('Finished with the catalog, removed a total of %d items' % removed)
    log.info("Length: %d, len(uids): %d, len(paths): %d" % (zcat._length.value,
        len(zcat.uids), len(zcat.paths)))

    transaction.commit()
Exemplo n.º 37
0
    def addColumn(self, name, default_value=None, threshold=10000):
        """Adds a row to the meta data schema"""
        schema = self.schema
        names = list(self.names)
        threshold = threshold if threshold is not None else 10000

        if name != name.strip():
            # Someone could have mistakenly added a space at the end
            # of the input field.
            LOG.warning('stripped space from new column %r -> %r', name,
                        name.strip())
            name = name.strip()

        if name in schema:
            raise CatalogError('The column %s already exists' % name)

        if name[0] == '_':
            raise CatalogError('Cannot cache fields beginning with "_"')

        values = schema.values()
        if values:
            schema[name] = max(values) + 1
        else:
            schema[name] = 0
        names.append(name)

        if default_value in (None, ''):
            default_value = MV

        if len(self):
            pghandler = ZLogHandler(threshold)
            pghandler.init('Adding %s column' % name, len(self))
            for i, (key, value) in enumerate(self.data.iteritems()):
                pghandler.report(i)
                self.data[key] = value + (default_value, )
            pghandler.finish()

        self.names = tuple(names)
        self.schema = schema

        # new column? update the brain
        self.updateBrains()
Exemplo n.º 38
0
def fix_themes_navigation(context):
    """ Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
        objects.
    """
    ctool = getToolByName(context, 'portal_catalog')
    brains = ctool.unrestrictedSearchResults(
        object_provides='eea.themecentre.interfaces.IThemeCentre')

    logger.info("Fixing eea.themecentre navigation ...")
    pghandler = ZLogHandler(100)
    pghandler.init('Fixing eea.themecentre navigation', len(brains))

    for index, brain in enumerate(brains):
        doc = brain.getObject()
        if not INavigationRoot.providedBy(doc):
            alsoProvides(doc, INavigationRoot)

        pghandler.report(index)
        if index % 100 == 0:
            transaction.savepoint(optimistic=True)

    transaction.savepoint(optimistic=True)
    pghandler.finish()
    logger.info("Fixing eea.themecentre navigation ... DONE")
Exemplo n.º 39
0
def agenda_item_project_set_summary_html(context):
    """Set the mimetype of the summary to text/html.

    Otherwise when editing you get a bare textarea and cannot save it.
    """
    catalog = getToolByName(context, 'portal_catalog')
    brains = catalog(portal_type='AgendaItemProject')
    num_objects = len(brains)
    logger.info("Updating agenda item summaries.")
    fixed = 0
    pghandler = ZLogHandler(100)
    pghandler.init('Setting mimetype html for agenda item summaries',
                   num_objects)
    for index, brain in enumerate(brains):
        pghandler.report(index)
        obj = brain.getObject()
        if obj.summary.mimetype != 'text/html':
            obj.summary.mimetype = 'text/html'
            fixed += 1
    pghandler.finish()
    logger.info("Have set summary mimetype to text/html for {0}/{1} agenda "
                "items.".format(fixed, len(brains)))
Exemplo n.º 40
0
def update_catalog_metadata(context, types=None):
    """Update catalog metadata.

    Adapted from updateIconMetadata in plone.app.upgrade v40/betas.py.
    """
    catalog = getToolByName(context, 'portal_catalog')
    search = catalog.unrestrictedSearchResults
    if types is None:
        logger.info('Updating catalog metadata.')
        brains = search(sort_on="path")
    else:
        if isinstance(types, basestring):
            types = [types]
        logger.info('Updating catalog metadata for %s.', ', '.join(types))
        brains = search(portal_type=types, sort_on="path")
    num_objects = len(brains)
    # Yes, this logs quite often, but I just know I am going to thank
    # myself for that...  Feedback about progress is good for my heart.
    # [Maurits]
    pghandler = ZLogHandler(10)
    pghandler.init('Updating catalog metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        obj = brain.getObject()
        # passing in a valid but inexpensive index, makes sure we don't
        # reindex the entire catalog including expensive indexes like
        # SearchableText
        brain_path = brain.getPath()
        try:
            catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
        except ConflictError:
            raise
        except Exception:
            pass
        i += 1
    pghandler.finish()
    logger.info('Done updating catalog metadata.')
Exemplo n.º 41
0
def fix_themes_navigation(context):
    """ Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
        objects.
    """
    ctool = getToolByName(context, 'portal_catalog')
    brains = ctool.unrestrictedSearchResults(
        object_provides='eea.themecentre.interfaces.IThemeCentre')

    logger.info("Fixing eea.themecentre navigation ...")
    pghandler = ZLogHandler(100)
    pghandler.init('Fixing eea.themecentre navigation', len(brains))

    for index, brain in enumerate(brains):
        doc = brain.getObject()
        if not INavigationRoot.providedBy(doc):
            alsoProvides(doc, INavigationRoot)

        pghandler.report(index)
        if index % 100 == 0:
            transaction.savepoint(optimistic=True)

    transaction.savepoint(optimistic=True)
    pghandler.finish()
    logger.info("Fixing eea.themecentre navigation ... DONE")
Exemplo n.º 42
0
def reindex_sortable_title(context):
    from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
    catalog = getToolByName(context, 'portal_catalog')
    _catalog = catalog._catalog
    indexes = _catalog.indexes
    sort_title_index = indexes.get('sortable_title', None)
    if sort_title_index is None:
        return
    from Products.PluginIndexes.FieldIndex import FieldIndex
    if not isinstance(sort_title_index, FieldIndex.FieldIndex):
        return
    change = []
    pghandler = ZLogHandler(10000)
    logger.info('Analyzing sortable_title index')
    pghandler.init('Analyzing sortable_title index', len(sort_title_index))
    for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
        pghandler.report(i)
        if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
            if hasattr(rids, 'keys'):
                change.extend(list(rids.keys()))
            else:
                change.append(rids)
    pghandler.finish()
    update_metadata = 'sortable_title' in _catalog.schema
    pghandler = ZLogHandler(1000)
    logger.info('Updating sortable_title index')
    pghandler.init('Updating sortable_title index', len(change))
    for i, rid in enumerate(change):
        pghandler.report(i)
        brain = _catalog[rid]
        try:
            obj = brain.getObject()
        except (AttributeError, KeyError):
            continue
        if update_metadata:
            obj.reindexObject()
        else:
            obj.reindexObject(idxs=['sortable_title'])
    pghandler.finish()
    logger.info('Updated `sortable_title` index.')
Exemplo n.º 43
0
def updateIconMetadata(context):
    """Update getIcon metadata column for all core content"""
    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon', None)
    typesToUpdate = {
        'Document' : ('document_icon.gif', 'document_icon.png'),
        'Event' : ('event_icon.gif', 'event_icon.png'),
        'File' : ('file_icon.gif', 'file_icon.png'),
        'Folder' : ('folder_icon.gif', 'folder_icon.png'),
        'Image' : ('image_icon.gif', 'image_icon.png'),
        'Link' : ('link_icon.gif', 'link_icon.png'),
        'News Item' : ('newsitem_icon.gif', 'newsitem_icon.png'),
        'Topic' : ('topic_icon.gif', 'topic_icon.png'),
    }
    ttool = getToolByName(context, 'portal_types')
    empty_icons = []
    for name in typesToUpdate.keys():
        fti = ttool.get(name)
        if fti:
            icon_expr = fti.getIconExprObject()
            if not icon_expr:
                empty_icons.append(name)

    brains = search(portal_type=empty_icons, sort_on="path")
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        if not brain_icon:
            continue
        old_icons = typesToUpdate[brain.portal_type]
        if getIconPos is not None:
            # if the old icon is a standard icon, we assume no customization
            # has taken place and we can simply empty the getIcon metadata
            # without loading the object
            new_value = ''
            if brain_icon not in old_icons:
                # Otherwise we need to ask the object
                new_value = ''
                obj = brain.getObject()
                method = getattr(aq_base(obj), 'getIcon', None)
                if method is not None:
                    try:
                        new_value = obj.getIcon
                        if callable(new_value):
                            new_value = new_value()
                    except ConflictError:
                        raise
                    except Exception:
                        new_value = ''
            if brain_icon != new_value:
                rid = brain.getRID()
                record = metadata[rid]
                new_record = list(record)
                new_record[getIconPos] = new_value
                metadata[rid] = tuple(new_record)
        else:
            # If we don't have a standard catalog tool, fall back to the
            # official API
            obj = brain.getObject()
            # passing in a valid but inexpensive index, makes sure we don't
            # reindex the entire catalog including expensive indexes like
            # SearchableText
            brain_path = brain.getPath()
            try:
                catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
            except ConflictError:
                raise
            except Exception:
                pass
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')