def uidDuplications(self):
    pc = getToolByName(self, 'portal_catalog')
    ptool = getToolByName(self, 'portal_url')
    pghandler = ZLogHandler(1000)
    portal = ptool.getPortalObject()
    basecat = pc._catalog
    index = basecat.indexes['UID']
    size = len(index._index.keys())
    pghandler.init('Remove stale items from catalog', size)
    cnt = 0
    for uid, rid in index._index.items():
        path = basecat.paths[rid]
        pghandler.report(cnt)
        cnt += 1
        try:
            obj = portal.restrictedTraverse(path)
        except:
            #            print "could not get obj for path %s, skipping" % path
            # we ignore this for the moment
            continue
        parent = aq_parent(obj)
        ids = [x['id'] for x in parent._objects]
        if obj.getId() not in ids and obj.getId() not in parent.objectIds():
            print "path %s not in parent " % path
            del index._index[uid]
            index._length.change(-1)

    return "done"
def uidDuplications(self):
    pc = getToolByName(self, 'portal_catalog')
    ptool = getToolByName(self, 'portal_url')
    pghandler = ZLogHandler(1000)
    portal = ptool.getPortalObject()
    basecat = pc._catalog
    index = basecat.indexes['UID']
    size = len(index._index.keys())
    pghandler.init('Remove stale items from catalog', size)
    cnt = 0
    for uid, rid in index._index.items():
        path = basecat.paths[rid]
        pghandler.report(cnt)
        cnt += 1
        try:
            obj = portal.restrictedTraverse(path)
        except:
#            print "could not get obj for path %s, skipping" % path
           # we ignore this for the moment
            continue
        parent = aq_parent(obj)
        ids = [x['id'] for x in parent._objects]
        if obj.getId() not in ids and obj.getId() not in parent.objectIds():
            print "path %s not in parent " % path
            del index._index[uid]
            index._length.change(-1)

    return "done"
Exemplo n.º 3
0
    def delColumn(self, name, threshold=10000):
        """Deletes a row from the meta data schema"""
        names = list(self.names)
        _index = names.index(name)
        threshold = threshold if threshold is not None else 10000

        if name not in self.schema:
            LOG.error('delColumn attempted to delete nonexistent '
                      'column %s.', str(name))
            return

        del names[_index]

        # rebuild the schema
        schema = {}
        for i, name in enumerate(names):
            schema[name] = i

        self.schema = schema
        self.names = tuple(names)

        # update the brain
        self.updateBrains()

        # remove the column value from each record
        if len(self):
            _next_index = _index + 1
            pghandler = ZLogHandler(threshold)
            pghandler.init('Deleting %s column' % name, len(self))
            for i, (key, value) in enumerate(self.data.iteritems()):
                pghandler.report(i)
                self.data[key] = value[:_index] + value[_next_index:]
            pghandler.finish()
Exemplo n.º 4
0
 def reindexIndexes(self, idxs=[], update_metadata=False, meta_types=[], portal_types=[]):
     """Reindex index including metadata if p_update_metadata=True.
        Filter meta_type/portal_type when p_meta_types and p_portal_types are given."""
     catalog = api.portal.get_tool('portal_catalog')
     paths = catalog._catalog.uids.keys()
     pghandler = ZLogHandler(steps=1000)
     i = 0
     pghandler.info(
         'In reindexIndexes, idxs={0}, update_metadata={1}, meta_types={2}, portal_types={3}'.format(
             repr(idxs), repr(update_metadata), repr(meta_types), repr(portal_types)))
     pghandler.init('reindexIndexes', len(paths))
     for p in paths:
         i += 1
         if pghandler:
             pghandler.report(i)
         obj = catalog.resolve_path(p)
         if obj is None:
             logger.error(
                 'reindexIndex could not resolve an object from the uid %r.' % p)
         else:
             if (meta_types and obj.meta_type not in meta_types) or \
                (portal_types and obj.portal_type not in portal_types):
                 continue
             catalog.catalog_object(
                 obj, p, idxs=idxs, update_metadata=update_metadata, pghandler=pghandler)
     if pghandler:
         pghandler.finish()
Exemplo n.º 5
0
def update_getIcon(context):
    typesToUpdate = {
        'Location': ('link_icon.gif', 'link_icon.png'),
        'Name': ('document_icon.gif', 'document_icon.png'),
        'PleiadesVocabulary': ('folder_icon.gif', 'folder_icon.png'),
        'PleiadesVocabularyTerm': ('document_icon.gif', 'document_icon.png'),
    }

    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon')

    brains = search(portal_type=typesToUpdate.keys(), sort_on="path")
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        old_icon, new_icon = typesToUpdate[brain.portal_type]
        if brain_icon != new_icon:
            rid = brain.getRID()
            record = metadata[rid]
            new_record = list(record)
            new_record[getIconPos] = new_icon
            metadata[rid] = tuple(new_record)
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')
Exemplo n.º 6
0
 def _cleanFTWLabels(self):
     """This fix partial migrations of stored ftw.labels:labeling that are
        still PersistentList and not PersistentMapping."""
     logger.info("Cleaning ftw.labels wrong annotations...")
     brains = self.catalog(object_provides=ILabelSupport.__identifier__)
     pghandler = ZLogHandler(steps=100)
     pghandler.init('Cleaning ftw.labels wrong annotations...', len(brains))
     i = 0
     cleaned = 0
     for brain in brains:
         i += 1
         pghandler.report(i)
         obj = brain.getObject()
         annotations = IAnnotations(obj)
         if 'ftw.labels:labeling' in annotations and \
            not isinstance(annotations['ftw.labels:labeling'], PersistentMapping):
             if annotations['ftw.labels:labeling']:
                 labeling = ILabeling(obj)
                 old_values = [label for label in labeling.storage]
                 del annotations['ftw.labels:labeling']
                 labeling._storage = None
                 labeling.update(old_values)
                 logger.info('In _cleanFTWLabels, cleaned %s' %
                             brain.getPath())
             else:
                 del annotations['ftw.labels:labeling']
             obj.reindexObject(idxs=['labels'])
             cleaned += 1
     if cleaned:
         self.warn(logger,
                   'In _cleanFTWLabels, cleaned %s element(s)' % cleaned)
     pghandler.finish()
     logger.info('Done.')
Exemplo n.º 7
0
    def delColumn(self, name, threshold=10000):
        """Deletes a row from the meta data schema"""
        names = list(self.names)
        _index = names.index(name)
        threshold = threshold if threshold is not None else 10000

        if name not in self.schema:
            LOG.error(
                'delColumn attempted to delete nonexistent '
                'column %s.', str(name))
            return

        del names[_index]

        # rebuild the schema
        schema = {}
        for i, name in enumerate(names):
            schema[name] = i

        self.schema = schema
        self.names = tuple(names)

        # update the brain
        self.updateBrains()

        # remove the column value from each record
        if len(self):
            _next_index = _index + 1
            pghandler = ZLogHandler(threshold)
            pghandler.init('Deleting %s column' % name, len(self))
            for i, (key, value) in enumerate(self.data.iteritems()):
                pghandler.report(i)
                self.data[key] = value[:_index] + value[_next_index:]
            pghandler.finish()
Exemplo n.º 8
0
def reindex_sortable_title(context):
    from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
    catalog = getToolByName(context, 'portal_catalog')
    _catalog = catalog._catalog
    indexes = _catalog.indexes
    sort_title_index = indexes.get('sortable_title', None)
    if sort_title_index is None:
        return
    from Products.PluginIndexes.FieldIndex import FieldIndex
    if not isinstance(sort_title_index, FieldIndex.FieldIndex):
        return
    change = []
    pghandler = ZLogHandler(10000)
    logger.info('Analyzing sortable_title index')
    pghandler.init('Analyzing sortable_title index', len(sort_title_index))
    for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
        pghandler.report(i)
        if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
            try:
                keys = rids.keys()
            except AttributeError:
                change.append(rids)
            else:
                change.extend(list(keys))
    pghandler.finish()
    # flake8 complained that reindex_sortable_title is too complex (11).
    # So we split it in two.
    _update_sortable_title(_catalog, change)
Exemplo n.º 9
0
    def _migrateLabelForCouncil(self):
        """Field labelForCouncil is replaced by
           otherMeetingConfigsClonableToFieldDetailedDescription in College and
           detailedDescription in Council."""
        logger.info(
            'Migrating field "labelForCouncil" in "meeting-config-college"...')
        # enable relevant fields in MeetingConfigs
        # College we use the "otherMeetingConfigsClonableToFieldLabelForCouncil" field
        # Council is correct
        college_cfg = self.tool.get('meeting-config-college')
        used_attrs = college_cfg.getUsedItemAttributes()
        used_attrs = replace_in_list(
            used_attrs, "labelForCouncil",
            "otherMeetingConfigsClonableToFieldLabelForCouncil")
        college_cfg.setUsedItemAttributes(used_attrs)
        logger.info('Done.')

        logger.info('Migrating field "labelForCouncil" for College items...')
        pghandler = ZLogHandler(steps=1000)
        brains = self.catalog(portal_type='MeetingItemCollege')
        pghandler.init('Migrating field "labelForCouncil" for College items',
                       len(brains))
        i = 0
        for brain in brains:
            i += 1
            pghandler.report(i)
            item = brain.getObject()
            if not item.fieldIsEmpty('labelForCouncil'):
                item.setOtherMeetingConfigsClonableToFieldLabelForCouncil(
                    item.getRawLabelForCouncil())
                item.setLabelForCouncil('')
        pghandler.finish()
        logger.info('Done.')
Exemplo n.º 10
0
def updateIconsInBrains(context, typesToUpdate=None):
    """Update getIcon metadata column in given types.

    typesToUpdate must be a dictionary, for example: {
        # portal_type: ('old_icon.gif', 'new_icon.png'),
        'Document': ('document_icon.gif', 'document_icon.png'),
        }

    The portal_types must have an empty icon_expr, because that is the
    main use case.
    """
    if not typesToUpdate:
        logger.warn('No typesToUpdate given for updateIconsInBrains.')
        return

    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon', None)
    empty_icons = _types_with_empty_icons(context, typesToUpdate)
    brains = search(portal_type=empty_icons, sort_on='path')
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        if not brain_icon:
            continue
        if getIconPos is not None:
            _update_icon_in_single_brain(brain, typesToUpdate, getIconPos,
                                         metadata)
        else:
            # If we don't have a standard catalog tool, fall back to the
            # official API
            obj = brain.getObject()
            # passing in a valid but inexpensive index, makes sure we don't
            # reindex the entire catalog including expensive indexes like
            # SearchableText
            brain_path = brain.getPath()
            try:
                catalog.catalog_object(obj, brain_path, ['id'], True,
                                       pghandler)
            except ConflictError:
                raise
            except Exception:
                pass
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')
Exemplo n.º 11
0
def updateIconsInBrains(context, typesToUpdate=None):
    """Update getIcon metadata column in given types.

    typesToUpdate must be a dictionary, for example: {
        # portal_type: ('old_icon.gif', 'new_icon.png'),
        'Document': ('document_icon.gif', 'document_icon.png'),
        }

    The portal_types must have an empty icon_expr, because that is the
    main use case.
    """
    if not typesToUpdate:
        logger.warn('No typesToUpdate given for updateIconsInBrains.')
        return

    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon', None)
    empty_icons = _types_with_empty_icons(context, typesToUpdate)
    brains = search(portal_type=empty_icons, sort_on='path')
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        if not brain_icon:
            continue
        if getIconPos is not None:
            _update_icon_in_single_brain(
                brain, typesToUpdate, getIconPos, metadata)
        else:
            # If we don't have a standard catalog tool, fall back to the
            # official API
            obj = brain.getObject()
            # passing in a valid but inexpensive index, makes sure we don't
            # reindex the entire catalog including expensive indexes like
            # SearchableText
            brain_path = brain.getPath()
            try:
                catalog.catalog_object(
                    obj, brain_path, ['id'], True, pghandler)
            except ConflictError:
                raise
            except Exception:
                pass
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')
Exemplo n.º 12
0
    def _migrate_categories_to_groups_in_charge_field(self):
        logger.info("Migrating Category to Groups in charge field")
        catalog = api.portal.get_tool("portal_catalog")
        own_org = get_own_organization()

        # First we create the appropriate groupsInCharge
        logger.info("Adapting MeetingConfig...")
        cfg = catalog(portal_type="MeetingConfig",
                      id=self.meeting_config_id)[0].getObject()
        categories = cfg.categories
        for cat_id, category in categories.contentItems():
            if cat_id not in own_org.objectIds():
                data = {
                    "id": cat_id,
                    "title": category.Title(),
                    "description": category.Description()
                }
                api.content.create(container=own_org,
                                   type="organization",
                                   **data)
            new_org_uid = org_id_to_uid(cat_id)

            enabled_plonegroup_org_uids = api.portal.get_registry_record(
                ORGANIZATIONS_REGISTRY)
            if new_org_uid not in enabled_plonegroup_org_uids:
                api.portal.set_registry_record(
                    ORGANIZATIONS_REGISTRY,
                    enabled_plonegroup_org_uids + [new_org_uid])

            ordered_group_in_charge_uids = cfg.getOrderedGroupsInCharge()
            if new_org_uid not in ordered_group_in_charge_uids:
                cfg.setOrderedGroupsInCharge(ordered_group_in_charge_uids +
                                             (new_org_uid, ))

            category.enabled = False

        # Next we migrate the category to groupsInCharge on MeetingItems
        brains = catalog(portal_type=(
            cfg.getItemTypeName(),
            cfg.getItemTypeName(configType="MeetingItemRecurring"),
            cfg.getItemTypeName(configType="MeetingItemTemplate"),
        ))
        pghandler = ZLogHandler(steps=1000)
        pghandler.init("Adapting items...", len(brains))
        for i, brain in enumerate(brains):
            pghandler.report(i)
            item = brain.getObject()
            if item.getCategory() != u"_none_":
                item.setGroupsInCharge([org_id_to_uid(item.getCategory())])
                item.setCategory(None)
                item.reindexObject(idxs=["getGroupsInCharge", "getCategory"])
        pghandler.finish()

        logger.info("Done migrating Categories to Groups in charge field")
Exemplo n.º 13
0
    def _migrateMeetingCategoryToDX(self):
        '''Migrate from AT MeetingCategory to DX meetingcategory.'''
        logger.info('Migrating MeetingCategory from AT to DX...')
        # update item classifier
        # migrate references
        pghandler = ZLogHandler(steps=100)
        brains = self.portal.reference_catalog(
            relationship='ItemClassification')
        pghandler.init('Updating field MeetingItem.classifier...', len(brains))
        pghandler.info('Updating field MeetingItem.classifier...')
        i = 0
        for brain in brains:
            i += 1
            pghandler.report(i)
            relation = brain.getObject()
            item = relation.getSourceObject()
            classifier = relation.getTargetObject()
            item.setClassifier(classifier.getId())
            item.reindexObject(idxs=['getRawClassifier'])
        # deleteReferences in a second phase
        for brain in brains:
            relation = brain.getObject()
            item = relation.getSourceObject()
            item.deleteReferences('ItemClassification')
        pghandler.finish()

        logger.info('Migrating categories and classifiers in configuration...')
        # make sure new portal_type meetingcategory is installed
        self.ps.runImportStepFromProfile(
            'profile-Products.PloneMeeting:default', 'typeinfo')
        # make sure no workflow used for meetingcategory
        self.wfTool.setChainForPortalTypes(('meetingcategory', ), ('', ))
        # adapt allowed_types for each MeetingConfig.categories/classifiers folders
        for cfg in self.tool.objectValues('MeetingConfig'):
            for folder_id in ('categories', 'classifiers'):
                constrain = IConstrainTypes(getattr(cfg, folder_id))
                constrain.setConstrainTypesMode(1)
                allowedTypes = ['meetingcategory']
                constrain.setLocallyAllowedTypes(allowedTypes)
                constrain.setImmediatelyAddableTypes(allowedTypes)
        # migrate to DX
        pac_migrate(self.portal, MeetingCategoryMigrator)
        self.removeUnusedPortalTypes(portal_types=['MeetingCategory'])
        # add meetingcategory to types_not_searched
        props = api.portal.get_tool('portal_properties').site_properties
        nsTypes = props.getProperty('types_not_searched')
        if 'meetingcategory' not in nsTypes:
            nsTypes = list(nsTypes)
            # MeetingCategory was removed by removeUnusedPortalTypes
            nsTypes.append('meetingcategory')
            props.manage_changeProperties(types_not_searched=tuple(nsTypes))
        logger.info('Done.')
Exemplo n.º 14
0
 def _umarkCreationFlagForEveryItems(self):
     """ """
     brains = self.catalog(meta_type='MeetingItem')
     pghandler = ZLogHandler(steps=100)
     pghandler.init('Cleaning ftw.labels wrong annotations...', len(brains))
     pghandler.info("Unmarking _at_creation_flag for every items...")
     i = 0
     for brain in brains:
         i += 1
         pghandler.report(i)
         item = brain.getObject()
         item.unmarkCreationFlag()
     pghandler.finish()
     logger.info('Done.')
Exemplo n.º 15
0
 def _removeMeetingItemGroupedItemsNumAttribute(self):
     """This field was supposed to be used but will not...
        Remove the MeetingItem.groupedItemsNum attribute on every existing items."""
     logger.info('Removing attribute "groupedItemsNum" from every items...')
     brains = self.catalog(meta_type="MeetingItem")
     pghandler = ZLogHandler(steps=1000)
     pghandler.init('Working', len(brains))
     i = 0
     for brain in brains:
         i += 1
         pghandler.report(i)
         item = brain.getObject()
         safe_delattr(item, "groupedItemsNum")
     pghandler.finish()
     logger.info('Done.')
Exemplo n.º 16
0
    def refresh_getIcon_catalog_metadata(context):
        """
        get_icon redefined: now boolean:
        needs to update metadata
        true if item is type image or has image field (named 'image')
        e.g. leadimage
        see https://github.com/plone/Products.CMFPlone/issues/1226
        """
        # Attention, this relies heavily on catalog internals.

        # get the more visible zcatalog
        zcatalog = getToolByName(context, 'portal_catalog')

        # get the more hidden, inner (real) catalog implementation
        catalog = zcatalog._catalog
        try:
            # Get the getIcon index value and
            # check if there is a getIcon at all, this may not exist in some
            # customizations, who knows, but always exists in default Plone
            metadata_index = catalog.names.index('getIcon')
        except ValueError:
            logger.info('`getIcon` not in metadata, skip upgrade step')
            return

        cnt = 0
        # search whole catalog
        results = zcatalog.unrestrictedSearchResults()
        num_objects = len(results)
        pghandler = ZLogHandler(1000)
        pghandler.init('Updating getIcon metadata', num_objects)
        for brain in results:
            pghandler.report(cnt)
            # First get the new value
            try:
                obj = brain._unrestrictedGetObject()
            except (AttributeError, KeyError, TypeError, NotFound):
                continue
            new_value = bool(getattr(aq_base(obj), 'image', False))

            # We can now update the record with the new getIcon value
            record = list(catalog.data[brain.getRID()])
            record[metadata_index] = new_value
            catalog.data[brain.getRID()] = tuple(record)

            cnt += 1  # we are curious
        # done
        pghandler.finish()
        logger.info('Reindexed `getIcon` for %s items', str(cnt))
Exemplo n.º 17
0
    def refresh_getIcon_catalog_metadata(context):
        """
        get_icon redefined: now boolean:
        needs to update metadata
        true if item is type image or has image field (named 'image')
        e.g. leadimage
        see https://github.com/plone/Products.CMFPlone/issues/1226
        """
        # Attention, this relies heavily on catalog internals.

        # get the more visible zcatalog
        zcatalog = getToolByName(context, 'portal_catalog')

        # get the more hidden, inner (real) catalog implementation
        catalog = zcatalog._catalog
        try:
            # Get the getIcon index value and
            # check if there is a getIcon at all, this may not exist in some
            # customizations, who knows, but always exists in default Plone
            metadata_index = catalog.names.index('getIcon')
        except ValueError:
            logger.info('`getIcon` not in metadata, skip upgrade step')
            return

        cnt = 0
        # search whole catalog
        results = zcatalog.unrestrictedSearchResults()
        num_objects = len(results)
        pghandler = ZLogHandler(1000)
        pghandler.init('Updating getIcon metadata', num_objects)
        for brain in results:
            pghandler.report(cnt)
            # First get the new value
            try:
                obj = brain._unrestrictedGetObject()
            except (AttributeError, KeyError, TypeError, NotFound):
                continue
            new_value = bool(getattr(aq_base(obj), 'image', False))

            # We can now update the record with the new getIcon value
            record = list(catalog.data[brain.getRID()])
            record[metadata_index] = new_value
            catalog.data[brain.getRID()] = tuple(record)

            cnt += 1  # we are curious
        # done
        pghandler.finish()
        logger.info('Reindexed `getIcon` for %s items' % str(cnt))
def cleanup(self):
    cat = getToolByName(self, "portal_catalog")
    purl = getToolByName(self, "portal_url")
    portal = purl.getPortalObject()
    zcat = cat._catalog
    pghandler = ZLogHandler(1000)
    size = len(zcat.uids)
    pghandler.init('Remove stale items from catalog', size)
    cnt = removed = 0
    for path in zcat.uids.keys():
        pghandler.report(cnt)
        cnt += 1
        try:
            obj = portal.restrictedTraverse(path)
        except:
            # XXX what to do?
            log.warning("no object found for %s" % path)
            cat.uncatalog_object(path)
            removed += 1
            log.info('Kill old path: %s' % path)
        else:
            if "/".join(obj.getPhysicalPath()) != path:
                # saved under an old path! kill it
                cat.uncatalog_object(path)
                #index = zcat.uids[path]
                #del zcat.uids[path]
                #del zcat.paths[index]
                #zcat._length.change(-1)
                removed += 1
                log.info('Kill old path: %s' % path)
            else:
                parent = aq_parent(obj)
                ids = [x['id'] for x in parent._objects]
                if obj.getId() not in ids and obj.getId(
                ) not in parent.objectIds():
                    cat.uncatalog_object(path)
                    removed += 1
                    log.info(
                        'ID not found as a child of current parent, kill it: %s'
                        % path)
    pghandler.finish()
    log.info('Finished with the catalog, removed a total of %d items' %
             removed)
    log.info("Length: %d, len(uids): %d, len(paths): %d" %
             (zcat._length.value, len(zcat.uids), len(zcat.paths)))

    transaction.commit()
Exemplo n.º 19
0
def _update_sortable_title(catalog, change):
    update_metadata = 'sortable_title' in catalog.schema
    pghandler = ZLogHandler(1000)
    logger.info('Updating sortable_title index')
    pghandler.init('Updating sortable_title index', len(change))
    for i, rid in enumerate(change):
        pghandler.report(i)
        brain = catalog[rid]
        try:
            obj = brain.getObject()
        except (AttributeError, KeyError):
            continue
        if update_metadata:
            obj.reindexObject()
        else:
            obj.reindexObject(idxs=['sortable_title'])
    pghandler.finish()
    logger.info('Updated `sortable_title` index.')
Exemplo n.º 20
0
def reindex_mime_type(context):
    # Attention, this relies heavily on catalog internals.
    # get the more visible zcatalog
    zcatalog = getToolByName(context, 'portal_catalog')
    # get the more hidden, inner (real) catalog implementation
    catalog = zcatalog._catalog
    try:
        metadata_index = catalog.names.index('mime_type')
    except ValueError:
        logger.info('`mime_type` not in metadata, skip upgrade step')
        return
    # we are interested in dexterity and archtetype images and files:
    ifaces = ['plone.app.contenttypes.interfaces.IImage',
              'plone.app.contenttypes.interfaces.IFile',
              'Products.ATContentTypes.interfaces.file.IFileContent',
              'Products.ATContentTypes.interfaces.image.IImageContent']
    cnt = 0
    results = zcatalog.unrestrictedSearchResults(object_provides=ifaces)
    num_objects = len(results)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating mime_type metadata', num_objects)
    for brain in results:
        pghandler.report(cnt)
        try:
            obj = brain._unrestrictedGetObject()
        except (AttributeError, KeyError, TypeError, NotFound):
            continue
        if not obj:
            continue
        # First get the new value:
        new_value = ''
        try:  # Dexterity
            new_value = obj.content_type()
        except TypeError:  # Archetypes
            new_value = obj.content_type
        except AttributeError:
            continue
        # We can now update the record with the new mime_type value
        record = list(catalog.data[brain.getRID()])
        record[metadata_index] = new_value
        catalog.data[brain.getRID()] = tuple(record)
        cnt += 1
    pghandler.finish()
    logger.info('Reindexed `mime_type` for %s items', str(cnt))
Exemplo n.º 21
0
def reindex_mime_type(context):
    # Attention, this relies heavily on catalog internals.
    # get the more visible zcatalog
    zcatalog = getToolByName(context, 'portal_catalog')
    # get the more hidden, inner (real) catalog implementation
    catalog = zcatalog._catalog
    try:
        metadata_index = catalog.names.index('mime_type')
    except ValueError:
        logger.info('`mime_type` not in metadata, skip upgrade step')
        return
    # we are interested in dexterity and archtetype images and files:
    ifaces = ['plone.app.contenttypes.interfaces.IImage',
              'plone.app.contenttypes.interfaces.IFile',
              'Products.ATContentTypes.interfaces.file.IFileContent',
              'Products.ATContentTypes.interfaces.image.IImageContent']
    cnt = 0
    results = zcatalog.unrestrictedSearchResults(object_provides=ifaces)
    num_objects = len(results)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating mime_type metadata', num_objects)
    for brain in results:
        pghandler.report(cnt)
        try:
            obj = brain._unrestrictedGetObject()
        except (AttributeError, KeyError, TypeError, NotFound):
            continue
        if not obj:
            continue
        # First get the new value:
        new_value = ''
        try:  # Dexterity
            new_value = obj.content_type()
        except TypeError:  # Archetypes
            new_value = obj.content_type
        except AttributeError:
            continue
        # We can now update the record with the new mime_type value
        record = list(catalog.data[brain.getRID()])
        record[metadata_index] = new_value
        catalog.data[brain.getRID()] = tuple(record)
        cnt += 1
    pghandler.finish()
    logger.info('Reindexed `mime_type` for %s items' % str(cnt))
def cleanup(self):
    cat = getToolByName(self, "portal_catalog")
    purl = getToolByName(self, "portal_url")
    portal = purl.getPortalObject()
    zcat = cat._catalog
    pghandler = ZLogHandler(1000)
    size = len(zcat.uids)
    pghandler.init('Remove stale items from catalog', size)
    cnt = removed = 0
    for path in zcat.uids.keys():
        pghandler.report(cnt)
        cnt += 1
        try:
            obj = portal.restrictedTraverse(path)
        except:
            # XXX what to do?
            log.warning("no object found for %s" % path)
            cat.uncatalog_object(path)
            removed += 1
            log.info('Kill old path: %s' % path)
        else:
            if  "/".join(obj.getPhysicalPath()) != path:
                # saved under an old path! kill it
                cat.uncatalog_object(path)
                #index = zcat.uids[path]
                #del zcat.uids[path]
                #del zcat.paths[index]
                #zcat._length.change(-1)
                removed += 1
                log.info('Kill old path: %s' % path)
            else:
                parent = aq_parent(obj)
                ids = [x['id'] for x in parent._objects]
                if obj.getId() not in ids and obj.getId() not in parent.objectIds():
                    cat.uncatalog_object(path)
                    removed += 1
                    log.info('ID not found as a child of current parent, kill it: %s' % path)
    pghandler.finish()
    log.info('Finished with the catalog, removed a total of %d items' % removed)
    log.info("Length: %d, len(uids): %d, len(paths): %d" % (zcat._length.value,
        len(zcat.uids), len(zcat.paths)))

    transaction.commit()
Exemplo n.º 23
0
def reindex_sortable_title(context):
    from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE

    catalog = getToolByName(context, "portal_catalog")
    _catalog = catalog._catalog
    indexes = _catalog.indexes
    sort_title_index = indexes.get("sortable_title", None)
    if sort_title_index is None:
        return
    from Products.PluginIndexes.FieldIndex import FieldIndex

    if not isinstance(sort_title_index, FieldIndex.FieldIndex):
        return
    change = []
    pghandler = ZLogHandler(10000)
    logger.info("Analyzing sortable_title index")
    pghandler.init("Analyzing sortable_title index", len(sort_title_index))
    for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
        pghandler.report(i)
        if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
            if hasattr(rids, "keys"):
                change.extend(list(rids.keys()))
            else:
                change.append(rids)
    pghandler.finish()
    update_metadata = "sortable_title" in _catalog.schema
    pghandler = ZLogHandler(1000)
    logger.info("Updating sortable_title index")
    pghandler.init("Updating sortable_title index", len(change))
    for i, rid in enumerate(change):
        pghandler.report(i)
        brain = _catalog[rid]
        try:
            obj = brain.getObject()
        except (AttributeError, KeyError):
            continue
        if update_metadata:
            obj.reindexObject()
        else:
            obj.reindexObject(idxs=["sortable_title"])
    pghandler.finish()
    logger.info("Updated `sortable_title` index.")
Exemplo n.º 24
0
    def addColumn(self, name, default_value=None, threshold=10000):
        """Adds a row to the meta data schema"""
        schema = self.schema
        names = list(self.names)
        threshold = threshold if threshold is not None else 10000

        if name != name.strip():
            # Someone could have mistakenly added a space at the end
            # of the input field.
            LOG.warning('stripped space from new column %r -> %r', name,
                        name.strip())
            name = name.strip()

        if name in schema:
            raise CatalogError('The column %s already exists' % name)

        if name[0] == '_':
            raise CatalogError('Cannot cache fields beginning with "_"')

        values = schema.values()
        if values:
            schema[name] = max(values) + 1
        else:
            schema[name] = 0
        names.append(name)

        if default_value in (None, ''):
            default_value = MV

        if len(self):
            pghandler = ZLogHandler(threshold)
            pghandler.init('Adding %s column' % name, len(self))
            for i, (key, value) in enumerate(self.data.iteritems()):
                pghandler.report(i)
                self.data[key] = value + (default_value, )
            pghandler.finish()

        self.names = tuple(names)
        self.schema = schema

        # new column? update the brain
        self.updateBrains()
Exemplo n.º 25
0
    def addColumn(self, name, default_value=None, threshold=10000):
        """Adds a row to the meta data schema"""
        schema = self.schema
        names = list(self.names)
        threshold = threshold if threshold is not None else 10000

        if name != name.strip():
            # Someone could have mistakenly added a space at the end
            # of the input field.
            LOG.warning('stripped space from new column %r -> %r', name,
                        name.strip())
            name = name.strip()

        if name in schema:
            raise CatalogError('The column %s already exists' % name)

        if name[0] == '_':
            raise CatalogError('Cannot cache fields beginning with "_"')

        values = schema.values()
        if values:
            schema[name] = max(values) + 1
        else:
            schema[name] = 0
        names.append(name)

        if default_value in (None, ''):
            default_value = MV

        if len(self):
            pghandler = ZLogHandler(threshold)
            pghandler.init('Adding %s column' % name, len(self))
            for i, (key, value) in enumerate(self.data.iteritems()):
                pghandler.report(i)
                self.data[key] = value + (default_value, )
            pghandler.finish()

        self.names = tuple(names)
        self.schema = schema

        # new column? update the brain
        self.updateBrains()
Exemplo n.º 26
0
 def reindexIndexesFor(self, idxs=[], **query):
     """ Reindex p_idxs on objects of given p_portal_types. """
     catalog = api.portal.get_tool('portal_catalog')
     brains = catalog(**query)
     pghandler = ZLogHandler(steps=1000)
     len_brains = len(brains)
     pghandler.info(
         'In reindexIndexesFor, reindexing indexes "{0}" on "{1}" objects ({2})...'.format(
             ', '.join(idxs) or '*',
             len(brains),
             str(query)))
     pghandler.init('reindexIndexesFor', len_brains)
     i = 0
     for brain in brains:
         i += 1
         pghandler.report(i)
         obj = brain.getObject()
         obj.reindexObject(idxs=idxs)
     pghandler.finish()
     logger.info('Done.')
Exemplo n.º 27
0
def reindex_sortable_title(context):
    from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
    catalog = getToolByName(context, 'portal_catalog')
    _catalog = catalog._catalog
    indexes = _catalog.indexes
    sort_title_index = indexes.get('sortable_title', None)
    if sort_title_index is None:
        return
    from Products.PluginIndexes.FieldIndex import FieldIndex
    if not isinstance(sort_title_index, FieldIndex.FieldIndex):
        return
    change = []
    pghandler = ZLogHandler(10000)
    logger.info('Analyzing sortable_title index')
    pghandler.init('Analyzing sortable_title index', len(sort_title_index))
    for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
        pghandler.report(i)
        if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
            if hasattr(rids, 'keys'):
                change.extend(list(rids.keys()))
            else:
                change.append(rids)
    pghandler.finish()
    update_metadata = 'sortable_title' in _catalog.schema
    pghandler = ZLogHandler(1000)
    logger.info('Updating sortable_title index')
    pghandler.init('Updating sortable_title index', len(change))
    for i, rid in enumerate(change):
        pghandler.report(i)
        brain = _catalog[rid]
        try:
            obj = brain.getObject()
        except (AttributeError, KeyError):
            continue
        if update_metadata:
            obj.reindexObject()
        else:
            obj.reindexObject(idxs=['sortable_title'])
    pghandler.finish()
    logger.info('Updated `sortable_title` index.')
Exemplo n.º 28
0
def agenda_item_project_set_summary_html(context):
    """Set the mimetype of the summary to text/html.

    Otherwise when editing you get a bare textarea and cannot save it.
    """
    catalog = getToolByName(context, 'portal_catalog')
    brains = catalog(portal_type='AgendaItemProject')
    num_objects = len(brains)
    logger.info("Updating agenda item summaries.")
    fixed = 0
    pghandler = ZLogHandler(100)
    pghandler.init('Setting mimetype html for agenda item summaries',
                   num_objects)
    for index, brain in enumerate(brains):
        pghandler.report(index)
        obj = brain.getObject()
        if obj.summary.mimetype != 'text/html':
            obj.summary.mimetype = 'text/html'
            fixed += 1
    pghandler.finish()
    logger.info("Have set summary mimetype to text/html for {0}/{1} agenda "
                "items.".format(fixed, len(brains)))
Exemplo n.º 29
0
def update_catalog_metadata(context, types=None):
    """Update catalog metadata.

    Adapted from updateIconMetadata in plone.app.upgrade v40/betas.py.
    """
    catalog = getToolByName(context, 'portal_catalog')
    search = catalog.unrestrictedSearchResults
    if types is None:
        logger.info('Updating catalog metadata.')
        brains = search(sort_on="path")
    else:
        if isinstance(types, basestring):
            types = [types]
        logger.info('Updating catalog metadata for %s.', ', '.join(types))
        brains = search(portal_type=types, sort_on="path")
    num_objects = len(brains)
    # Yes, this logs quite often, but I just know I am going to thank
    # myself for that...  Feedback about progress is good for my heart.
    # [Maurits]
    pghandler = ZLogHandler(10)
    pghandler.init('Updating catalog metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        obj = brain.getObject()
        # passing in a valid but inexpensive index, makes sure we don't
        # reindex the entire catalog including expensive indexes like
        # SearchableText
        brain_path = brain.getPath()
        try:
            catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
        except ConflictError:
            raise
        except Exception:
            pass
        i += 1
    pghandler.finish()
    logger.info('Done updating catalog metadata.')
Exemplo n.º 30
0
def fix_themes_navigation(context):
    """ Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
        objects.
    """
    ctool = getToolByName(context, 'portal_catalog')
    brains = ctool.unrestrictedSearchResults(
        object_provides='eea.themecentre.interfaces.IThemeCentre')

    logger.info("Fixing eea.themecentre navigation ...")
    pghandler = ZLogHandler(100)
    pghandler.init('Fixing eea.themecentre navigation', len(brains))

    for index, brain in enumerate(brains):
        doc = brain.getObject()
        if not INavigationRoot.providedBy(doc):
            alsoProvides(doc, INavigationRoot)

        pghandler.report(index)
        if index % 100 == 0:
            transaction.savepoint(optimistic=True)

    transaction.savepoint(optimistic=True)
    pghandler.finish()
    logger.info("Fixing eea.themecentre navigation ... DONE")
Exemplo n.º 31
0
def fix_themes_navigation(context):
    """ Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
        objects.
    """
    ctool = getToolByName(context, 'portal_catalog')
    brains = ctool.unrestrictedSearchResults(
        object_provides='eea.themecentre.interfaces.IThemeCentre')

    logger.info("Fixing eea.themecentre navigation ...")
    pghandler = ZLogHandler(100)
    pghandler.init('Fixing eea.themecentre navigation', len(brains))

    for index, brain in enumerate(brains):
        doc = brain.getObject()
        if not INavigationRoot.providedBy(doc):
            alsoProvides(doc, INavigationRoot)

        pghandler.report(index)
        if index % 100 == 0:
            transaction.savepoint(optimistic=True)

    transaction.savepoint(optimistic=True)
    pghandler.finish()
    logger.info("Fixing eea.themecentre navigation ... DONE")
Exemplo n.º 32
0
def updateIconMetadata(context):
    """Update getIcon metadata column for all core content"""
    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon', None)
    typesToUpdate = {
        'Document': ('document_icon.gif', 'document_icon.png'),
        'Event': ('event_icon.gif', 'event_icon.png'),
        'File': ('file_icon.gif', 'file_icon.png'),
        'Folder': ('folder_icon.gif', 'folder_icon.png'),
        'Image': ('image_icon.gif', 'image_icon.png'),
        'Link': ('link_icon.gif', 'link_icon.png'),
        'News Item': ('newsitem_icon.gif', 'newsitem_icon.png'),
        'Topic': ('topic_icon.gif', 'topic_icon.png'),
    }
    ttool = getToolByName(context, 'portal_types')
    empty_icons = []
    for name in typesToUpdate.keys():
        fti = ttool.get(name)
        if fti:
            icon_expr = fti.getIconExprObject()
            if not icon_expr:
                empty_icons.append(name)

    brains = search(portal_type=empty_icons, sort_on="path")
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        if not brain_icon:
            continue
        old_icons = typesToUpdate[brain.portal_type]
        if getIconPos is not None:
            # if the old icon is a standard icon, we assume no customization
            # has taken place and we can simply empty the getIcon metadata
            # without loading the object
            new_value = ''
            if brain_icon not in old_icons:
                # Otherwise we need to ask the object
                new_value = ''
                obj = brain.getObject()
                method = getattr(aq_base(obj), 'getIcon', None)
                if method is not None:
                    try:
                        new_value = obj.getIcon
                        if callable(new_value):
                            new_value = new_value()
                    except ConflictError:
                        raise
                    except Exception:
                        new_value = ''
            if brain_icon != new_value:
                rid = brain.getRID()
                record = metadata[rid]
                new_record = list(record)
                new_record[getIconPos] = new_value
                metadata[rid] = tuple(new_record)
        else:
            # If we don't have a standard catalog tool, fall back to the
            # official API
            obj = brain.getObject()
            # passing in a valid but inexpensive index, makes sure we don't
            # reindex the entire catalog including expensive indexes like
            # SearchableText
            brain_path = brain.getPath()
            try:
                catalog.catalog_object(obj, brain_path, ['id'], True,
                                       pghandler)
            except ConflictError:
                raise
            except Exception:
                pass
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')
Exemplo n.º 33
0
def updateIconMetadata(context):
    """Update getIcon metadata column for all core content"""
    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon', None)
    typesToUpdate = {
        'Document' : ('document_icon.gif', 'document_icon.png'),
        'Event' : ('event_icon.gif', 'event_icon.png'),
        'File' : ('file_icon.gif', 'file_icon.png'),
        'Folder' : ('folder_icon.gif', 'folder_icon.png'),
        'Image' : ('image_icon.gif', 'image_icon.png'),
        'Link' : ('link_icon.gif', 'link_icon.png'),
        'News Item' : ('newsitem_icon.gif', 'newsitem_icon.png'),
        'Topic' : ('topic_icon.gif', 'topic_icon.png'),
    }
    ttool = getToolByName(context, 'portal_types')
    empty_icons = []
    for name in typesToUpdate.keys():
        fti = ttool.get(name)
        if fti:
            icon_expr = fti.getIconExprObject()
            if not icon_expr:
                empty_icons.append(name)

    brains = search(portal_type=empty_icons, sort_on="path")
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        if not brain_icon:
            continue
        old_icons = typesToUpdate[brain.portal_type]
        if getIconPos is not None:
            # if the old icon is a standard icon, we assume no customization
            # has taken place and we can simply empty the getIcon metadata
            # without loading the object
            new_value = ''
            if brain_icon not in old_icons:
                # Otherwise we need to ask the object
                new_value = ''
                obj = brain.getObject()
                method = getattr(aq_base(obj), 'getIcon', None)
                if method is not None:
                    try:
                        new_value = obj.getIcon
                        if callable(new_value):
                            new_value = new_value()
                    except ConflictError:
                        raise
                    except Exception:
                        new_value = ''
            if brain_icon != new_value:
                rid = brain.getRID()
                record = metadata[rid]
                new_record = list(record)
                new_record[getIconPos] = new_value
                metadata[rid] = tuple(new_record)
        else:
            # If we don't have a standard catalog tool, fall back to the
            # official API
            obj = brain.getObject()
            # passing in a valid but inexpensive index, makes sure we don't
            # reindex the entire catalog including expensive indexes like
            # SearchableText
            brain_path = brain.getPath()
            try:
                catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
            except ConflictError:
                raise
            except Exception:
                pass
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')