예제 #1
0
    def delColumn(self, name, threshold=10000):
        """Deletes a row from the meta data schema"""
        names = list(self.names)
        _index = names.index(name)
        threshold = threshold if threshold is not None else 10000

        if name not in self.schema:
            LOG.error('delColumn attempted to delete nonexistent '
                      'column %s.', str(name))
            return

        del names[_index]

        # rebuild the schema
        schema = {}
        for i, name in enumerate(names):
            schema[name] = i

        self.schema = schema
        self.names = tuple(names)

        # update the brain
        self.updateBrains()

        # remove the column value from each record
        if len(self):
            _next_index = _index + 1
            pghandler = ZLogHandler(threshold)
            pghandler.init('Deleting %s column' % name, len(self))
            for i, (key, value) in enumerate(self.data.iteritems()):
                pghandler.report(i)
                self.data[key] = value[:_index] + value[_next_index:]
            pghandler.finish()
예제 #2
0
def update_getIcon(context):
    typesToUpdate = {
        'Location': ('link_icon.gif', 'link_icon.png'),
        'Name': ('document_icon.gif', 'document_icon.png'),
        'PleiadesVocabulary': ('folder_icon.gif', 'folder_icon.png'),
        'PleiadesVocabularyTerm': ('document_icon.gif', 'document_icon.png'),
    }

    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon')

    brains = search(portal_type=typesToUpdate.keys(), sort_on="path")
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        old_icon, new_icon = typesToUpdate[brain.portal_type]
        if brain_icon != new_icon:
            rid = brain.getRID()
            record = metadata[rid]
            new_record = list(record)
            new_record[getIconPos] = new_icon
            metadata[rid] = tuple(new_record)
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')
def uidDuplications(self):
    pc = getToolByName(self, 'portal_catalog')
    ptool = getToolByName(self, 'portal_url')
    pghandler = ZLogHandler(1000)
    portal = ptool.getPortalObject()
    basecat = pc._catalog
    index = basecat.indexes['UID']
    size = len(index._index.keys())
    pghandler.init('Remove stale items from catalog', size)
    cnt = 0
    for uid, rid in index._index.items():
        path = basecat.paths[rid]
        pghandler.report(cnt)
        cnt += 1
        try:
            obj = portal.restrictedTraverse(path)
        except:
#            print "could not get obj for path %s, skipping" % path
           # we ignore this for the moment
            continue
        parent = aq_parent(obj)
        ids = [x['id'] for x in parent._objects]
        if obj.getId() not in ids and obj.getId() not in parent.objectIds():
            print "path %s not in parent " % path
            del index._index[uid]
            index._length.change(-1)

    return "done"
예제 #4
0
def updateIconsInBrains(context, typesToUpdate=None):
    """Update getIcon metadata column in given types.

    typesToUpdate must be a dictionary, for example: {
        # portal_type: ('old_icon.gif', 'new_icon.png'),
        'Document': ('document_icon.gif', 'document_icon.png'),
        }

    The portal_types must have an empty icon_expr, because that is the
    main use case.
    """
    if not typesToUpdate:
        logger.warn('No typesToUpdate given for updateIconsInBrains.')
        return

    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon', None)
    empty_icons = _types_with_empty_icons(context, typesToUpdate)
    brains = search(portal_type=empty_icons, sort_on='path')
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        if not brain_icon:
            continue
        if getIconPos is not None:
            _update_icon_in_single_brain(
                brain, typesToUpdate, getIconPos, metadata)
        else:
            # If we don't have a standard catalog tool, fall back to the
            # official API
            obj = brain.getObject()
            # passing in a valid but inexpensive index, makes sure we don't
            # reindex the entire catalog including expensive indexes like
            # SearchableText
            brain_path = brain.getPath()
            try:
                catalog.catalog_object(
                    obj, brain_path, ['id'], True, pghandler)
            except ConflictError:
                raise
            except Exception:
                pass
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')
예제 #5
0
    def refresh_getIcon_catalog_metadata(context):
        """
        get_icon redefined: now boolean:
        needs to update metadata
        true if item is type image or has image field (named 'image')
        e.g. leadimage
        see https://github.com/plone/Products.CMFPlone/issues/1226
        """
        # Attention, this relies heavily on catalog internals.

        # get the more visible zcatalog
        zcatalog = getToolByName(context, 'portal_catalog')

        # get the more hidden, inner (real) catalog implementation
        catalog = zcatalog._catalog
        try:
            # Get the getIcon index value and
            # check if there is a getIcon at all, this may not exist in some
            # customizations, who knows, but always exists in default Plone
            metadata_index = catalog.names.index('getIcon')
        except ValueError:
            logger.info('`getIcon` not in metadata, skip upgrade step')
            return

        cnt = 0
        # search whole catalog
        results = zcatalog.unrestrictedSearchResults()
        num_objects = len(results)
        pghandler = ZLogHandler(1000)
        pghandler.init('Updating getIcon metadata', num_objects)
        for brain in results:
            pghandler.report(cnt)
            # First get the new value
            try:
                obj = brain._unrestrictedGetObject()
            except (AttributeError, KeyError, TypeError, NotFound):
                continue
            new_value = bool(getattr(aq_base(obj), 'image', False))

            # We can now update the record with the new getIcon value
            record = list(catalog.data[brain.getRID()])
            record[metadata_index] = new_value
            catalog.data[brain.getRID()] = tuple(record)

            cnt += 1  # we are curious
        # done
        pghandler.finish()
        logger.info('Reindexed `getIcon` for %s items', str(cnt))
예제 #6
0
def reindex_mime_type(context):
    # Attention, this relies heavily on catalog internals.
    # get the more visible zcatalog
    zcatalog = getToolByName(context, 'portal_catalog')
    # get the more hidden, inner (real) catalog implementation
    catalog = zcatalog._catalog
    try:
        metadata_index = catalog.names.index('mime_type')
    except ValueError:
        logger.info('`mime_type` not in metadata, skip upgrade step')
        return
    # we are interested in dexterity and archtetype images and files:
    ifaces = ['plone.app.contenttypes.interfaces.IImage',
              'plone.app.contenttypes.interfaces.IFile',
              'Products.ATContentTypes.interfaces.file.IFileContent',
              'Products.ATContentTypes.interfaces.image.IImageContent']
    cnt = 0
    results = zcatalog.unrestrictedSearchResults(object_provides=ifaces)
    num_objects = len(results)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating mime_type metadata', num_objects)
    for brain in results:
        pghandler.report(cnt)
        try:
            obj = brain._unrestrictedGetObject()
        except (AttributeError, KeyError, TypeError, NotFound):
            continue
        if not obj:
            continue
        # First get the new value:
        new_value = ''
        try:  # Dexterity
            new_value = obj.content_type()
        except TypeError:  # Archetypes
            new_value = obj.content_type
        except AttributeError:
            continue
        # We can now update the record with the new mime_type value
        record = list(catalog.data[brain.getRID()])
        record[metadata_index] = new_value
        catalog.data[brain.getRID()] = tuple(record)
        cnt += 1
    pghandler.finish()
    logger.info('Reindexed `mime_type` for %s items', str(cnt))
def cleanup(self):
    cat = getToolByName(self, "portal_catalog")
    purl = getToolByName(self, "portal_url")
    portal = purl.getPortalObject()
    zcat = cat._catalog
    pghandler = ZLogHandler(1000)
    size = len(zcat.uids)
    pghandler.init('Remove stale items from catalog', size)
    cnt = removed = 0
    for path in zcat.uids.keys():
        pghandler.report(cnt)
        cnt += 1
        try:
            obj = portal.restrictedTraverse(path)
        except:
            # XXX what to do?
            log.warning("no object found for %s" % path)
            cat.uncatalog_object(path)
            removed += 1
            log.info('Kill old path: %s' % path)
        else:
            if  "/".join(obj.getPhysicalPath()) != path:
                # saved under an old path! kill it
                cat.uncatalog_object(path)
                #index = zcat.uids[path]
                #del zcat.uids[path]
                #del zcat.paths[index]
                #zcat._length.change(-1)
                removed += 1
                log.info('Kill old path: %s' % path)
            else:
                parent = aq_parent(obj)
                ids = [x['id'] for x in parent._objects]
                if obj.getId() not in ids and obj.getId() not in parent.objectIds():
                    cat.uncatalog_object(path)
                    removed += 1
                    log.info('ID not found as a child of current parent, kill it: %s' % path)
    pghandler.finish()
    log.info('Finished with the catalog, removed a total of %d items' % removed)
    log.info("Length: %d, len(uids): %d, len(paths): %d" % (zcat._length.value,
        len(zcat.uids), len(zcat.paths)))

    transaction.commit()
예제 #8
0
    def addColumn(self, name, default_value=None, threshold=10000):
        """Adds a row to the meta data schema"""
        schema = self.schema
        names = list(self.names)
        threshold = threshold if threshold is not None else 10000

        if name != name.strip():
            # Someone could have mistakenly added a space at the end
            # of the input field.
            LOG.warning('stripped space from new column %r -> %r', name,
                        name.strip())
            name = name.strip()

        if name in schema:
            raise CatalogError('The column %s already exists' % name)

        if name[0] == '_':
            raise CatalogError('Cannot cache fields beginning with "_"')

        values = schema.values()
        if values:
            schema[name] = max(values) + 1
        else:
            schema[name] = 0
        names.append(name)

        if default_value in (None, ''):
            default_value = MV

        if len(self):
            pghandler = ZLogHandler(threshold)
            pghandler.init('Adding %s column' % name, len(self))
            for i, (key, value) in enumerate(self.data.iteritems()):
                pghandler.report(i)
                self.data[key] = value + (default_value, )
            pghandler.finish()

        self.names = tuple(names)
        self.schema = schema

        # new column? update the brain
        self.updateBrains()
예제 #9
0
파일: alphas.py 프로젝트: pigaov10/plone4.3
def reindex_sortable_title(context):
    from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE

    catalog = getToolByName(context, "portal_catalog")
    _catalog = catalog._catalog
    indexes = _catalog.indexes
    sort_title_index = indexes.get("sortable_title", None)
    if sort_title_index is None:
        return
    from Products.PluginIndexes.FieldIndex import FieldIndex

    if not isinstance(sort_title_index, FieldIndex.FieldIndex):
        return
    change = []
    pghandler = ZLogHandler(10000)
    logger.info("Analyzing sortable_title index")
    pghandler.init("Analyzing sortable_title index", len(sort_title_index))
    for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
        pghandler.report(i)
        if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
            if hasattr(rids, "keys"):
                change.extend(list(rids.keys()))
            else:
                change.append(rids)
    pghandler.finish()
    update_metadata = "sortable_title" in _catalog.schema
    pghandler = ZLogHandler(1000)
    logger.info("Updating sortable_title index")
    pghandler.init("Updating sortable_title index", len(change))
    for i, rid in enumerate(change):
        pghandler.report(i)
        brain = _catalog[rid]
        try:
            obj = brain.getObject()
        except (AttributeError, KeyError):
            continue
        if update_metadata:
            obj.reindexObject()
        else:
            obj.reindexObject(idxs=["sortable_title"])
    pghandler.finish()
    logger.info("Updated `sortable_title` index.")
예제 #10
0
def reindex_sortable_title(context):
    from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE
    catalog = getToolByName(context, 'portal_catalog')
    _catalog = catalog._catalog
    indexes = _catalog.indexes
    sort_title_index = indexes.get('sortable_title', None)
    if sort_title_index is None:
        return
    from Products.PluginIndexes.FieldIndex import FieldIndex
    if not isinstance(sort_title_index, FieldIndex.FieldIndex):
        return
    change = []
    pghandler = ZLogHandler(10000)
    logger.info('Analyzing sortable_title index')
    pghandler.init('Analyzing sortable_title index', len(sort_title_index))
    for i, (name, rids) in enumerate(sort_title_index._index.iteritems()):
        pghandler.report(i)
        if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name):
            if hasattr(rids, 'keys'):
                change.extend(list(rids.keys()))
            else:
                change.append(rids)
    pghandler.finish()
    update_metadata = 'sortable_title' in _catalog.schema
    pghandler = ZLogHandler(1000)
    logger.info('Updating sortable_title index')
    pghandler.init('Updating sortable_title index', len(change))
    for i, rid in enumerate(change):
        pghandler.report(i)
        brain = _catalog[rid]
        try:
            obj = brain.getObject()
        except (AttributeError, KeyError):
            continue
        if update_metadata:
            obj.reindexObject()
        else:
            obj.reindexObject(idxs=['sortable_title'])
    pghandler.finish()
    logger.info('Updated `sortable_title` index.')
예제 #11
0
def agenda_item_project_set_summary_html(context):
    """Set the mimetype of the summary to text/html.

    Otherwise when editing you get a bare textarea and cannot save it.
    """
    catalog = getToolByName(context, 'portal_catalog')
    brains = catalog(portal_type='AgendaItemProject')
    num_objects = len(brains)
    logger.info("Updating agenda item summaries.")
    fixed = 0
    pghandler = ZLogHandler(100)
    pghandler.init('Setting mimetype html for agenda item summaries',
                   num_objects)
    for index, brain in enumerate(brains):
        pghandler.report(index)
        obj = brain.getObject()
        if obj.summary.mimetype != 'text/html':
            obj.summary.mimetype = 'text/html'
            fixed += 1
    pghandler.finish()
    logger.info("Have set summary mimetype to text/html for {0}/{1} agenda "
                "items.".format(fixed, len(brains)))
예제 #12
0
def update_catalog_metadata(context, types=None):
    """Update catalog metadata.

    Adapted from updateIconMetadata in plone.app.upgrade v40/betas.py.
    """
    catalog = getToolByName(context, 'portal_catalog')
    search = catalog.unrestrictedSearchResults
    if types is None:
        logger.info('Updating catalog metadata.')
        brains = search(sort_on="path")
    else:
        if isinstance(types, basestring):
            types = [types]
        logger.info('Updating catalog metadata for %s.', ', '.join(types))
        brains = search(portal_type=types, sort_on="path")
    num_objects = len(brains)
    # Yes, this logs quite often, but I just know I am going to thank
    # myself for that...  Feedback about progress is good for my heart.
    # [Maurits]
    pghandler = ZLogHandler(10)
    pghandler.init('Updating catalog metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        obj = brain.getObject()
        # passing in a valid but inexpensive index, makes sure we don't
        # reindex the entire catalog including expensive indexes like
        # SearchableText
        brain_path = brain.getPath()
        try:
            catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
        except ConflictError:
            raise
        except Exception:
            pass
        i += 1
    pghandler.finish()
    logger.info('Done updating catalog metadata.')
예제 #13
0
def fix_themes_navigation(context):
    """ Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
        objects.
    """
    ctool = getToolByName(context, 'portal_catalog')
    brains = ctool.unrestrictedSearchResults(
        object_provides='eea.themecentre.interfaces.IThemeCentre')

    logger.info("Fixing eea.themecentre navigation ...")
    pghandler = ZLogHandler(100)
    pghandler.init('Fixing eea.themecentre navigation', len(brains))

    for index, brain in enumerate(brains):
        doc = brain.getObject()
        if not INavigationRoot.providedBy(doc):
            alsoProvides(doc, INavigationRoot)

        pghandler.report(index)
        if index % 100 == 0:
            transaction.savepoint(optimistic=True)

    transaction.savepoint(optimistic=True)
    pghandler.finish()
    logger.info("Fixing eea.themecentre navigation ... DONE")
예제 #14
0
def fix_themes_navigation(context):
    """ Add INavigationRoot marker interface for eea.themecentre IThemeTaggable
        objects.
    """
    ctool = getToolByName(context, 'portal_catalog')
    brains = ctool.unrestrictedSearchResults(
        object_provides='eea.themecentre.interfaces.IThemeCentre')

    logger.info("Fixing eea.themecentre navigation ...")
    pghandler = ZLogHandler(100)
    pghandler.init('Fixing eea.themecentre navigation', len(brains))

    for index, brain in enumerate(brains):
        doc = brain.getObject()
        if not INavigationRoot.providedBy(doc):
            alsoProvides(doc, INavigationRoot)

        pghandler.report(index)
        if index % 100 == 0:
            transaction.savepoint(optimistic=True)

    transaction.savepoint(optimistic=True)
    pghandler.finish()
    logger.info("Fixing eea.themecentre navigation ... DONE")
예제 #15
0
def updateIconMetadata(context):
    """Update getIcon metadata column for all core content"""
    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon', None)
    typesToUpdate = {
        'Document': ('document_icon.gif', 'document_icon.png'),
        'Event': ('event_icon.gif', 'event_icon.png'),
        'File': ('file_icon.gif', 'file_icon.png'),
        'Folder': ('folder_icon.gif', 'folder_icon.png'),
        'Image': ('image_icon.gif', 'image_icon.png'),
        'Link': ('link_icon.gif', 'link_icon.png'),
        'News Item': ('newsitem_icon.gif', 'newsitem_icon.png'),
        'Topic': ('topic_icon.gif', 'topic_icon.png'),
    }
    ttool = getToolByName(context, 'portal_types')
    empty_icons = []
    for name in typesToUpdate.keys():
        fti = ttool.get(name)
        if fti:
            icon_expr = fti.getIconExprObject()
            if not icon_expr:
                empty_icons.append(name)

    brains = search(portal_type=empty_icons, sort_on="path")
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        if not brain_icon:
            continue
        old_icons = typesToUpdate[brain.portal_type]
        if getIconPos is not None:
            # if the old icon is a standard icon, we assume no customization
            # has taken place and we can simply empty the getIcon metadata
            # without loading the object
            new_value = ''
            if brain_icon not in old_icons:
                # Otherwise we need to ask the object
                new_value = ''
                obj = brain.getObject()
                method = getattr(aq_base(obj), 'getIcon', None)
                if method is not None:
                    try:
                        new_value = obj.getIcon
                        if callable(new_value):
                            new_value = new_value()
                    except ConflictError:
                        raise
                    except Exception:
                        new_value = ''
            if brain_icon != new_value:
                rid = brain.getRID()
                record = metadata[rid]
                new_record = list(record)
                new_record[getIconPos] = new_value
                metadata[rid] = tuple(new_record)
        else:
            # If we don't have a standard catalog tool, fall back to the
            # official API
            obj = brain.getObject()
            # passing in a valid but inexpensive index, makes sure we don't
            # reindex the entire catalog including expensive indexes like
            # SearchableText
            brain_path = brain.getPath()
            try:
                catalog.catalog_object(obj, brain_path, ['id'], True,
                                       pghandler)
            except ConflictError:
                raise
            except Exception:
                pass
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')
예제 #16
0
def updateIconMetadata(context):
    """Update getIcon metadata column for all core content"""
    catalog = getToolByName(context, 'portal_catalog')
    logger.info('Updating `getIcon` metadata.')
    search = catalog.unrestrictedSearchResults
    _catalog = getattr(catalog, '_catalog', None)
    getIconPos = None
    if _catalog is not None:
        metadata = _catalog.data
        getIconPos = _catalog.schema.get('getIcon', None)
    typesToUpdate = {
        'Document' : ('document_icon.gif', 'document_icon.png'),
        'Event' : ('event_icon.gif', 'event_icon.png'),
        'File' : ('file_icon.gif', 'file_icon.png'),
        'Folder' : ('folder_icon.gif', 'folder_icon.png'),
        'Image' : ('image_icon.gif', 'image_icon.png'),
        'Link' : ('link_icon.gif', 'link_icon.png'),
        'News Item' : ('newsitem_icon.gif', 'newsitem_icon.png'),
        'Topic' : ('topic_icon.gif', 'topic_icon.png'),
    }
    ttool = getToolByName(context, 'portal_types')
    empty_icons = []
    for name in typesToUpdate.keys():
        fti = ttool.get(name)
        if fti:
            icon_expr = fti.getIconExprObject()
            if not icon_expr:
                empty_icons.append(name)

    brains = search(portal_type=empty_icons, sort_on="path")
    num_objects = len(brains)
    pghandler = ZLogHandler(1000)
    pghandler.init('Updating getIcon metadata', num_objects)
    i = 0
    for brain in brains:
        pghandler.report(i)
        brain_icon = brain.getIcon
        if not brain_icon:
            continue
        old_icons = typesToUpdate[brain.portal_type]
        if getIconPos is not None:
            # if the old icon is a standard icon, we assume no customization
            # has taken place and we can simply empty the getIcon metadata
            # without loading the object
            new_value = ''
            if brain_icon not in old_icons:
                # Otherwise we need to ask the object
                new_value = ''
                obj = brain.getObject()
                method = getattr(aq_base(obj), 'getIcon', None)
                if method is not None:
                    try:
                        new_value = obj.getIcon
                        if callable(new_value):
                            new_value = new_value()
                    except ConflictError:
                        raise
                    except Exception:
                        new_value = ''
            if brain_icon != new_value:
                rid = brain.getRID()
                record = metadata[rid]
                new_record = list(record)
                new_record[getIconPos] = new_value
                metadata[rid] = tuple(new_record)
        else:
            # If we don't have a standard catalog tool, fall back to the
            # official API
            obj = brain.getObject()
            # passing in a valid but inexpensive index, makes sure we don't
            # reindex the entire catalog including expensive indexes like
            # SearchableText
            brain_path = brain.getPath()
            try:
                catalog.catalog_object(obj, brain_path, ['id'], True, pghandler)
            except ConflictError:
                raise
            except Exception:
                pass
        i += 1
    pghandler.finish()
    logger.info('Updated `getIcon` metadata.')