def delColumn(self, name, threshold=10000): """Deletes a row from the meta data schema""" names = list(self.names) _index = names.index(name) threshold = threshold if threshold is not None else 10000 if name not in self.schema: LOG.error('delColumn attempted to delete nonexistent ' 'column %s.', str(name)) return del names[_index] # rebuild the schema schema = {} for i, name in enumerate(names): schema[name] = i self.schema = schema self.names = tuple(names) # update the brain self.updateBrains() # remove the column value from each record if len(self): _next_index = _index + 1 pghandler = ZLogHandler(threshold) pghandler.init('Deleting %s column' % name, len(self)) for i, (key, value) in enumerate(self.data.iteritems()): pghandler.report(i) self.data[key] = value[:_index] + value[_next_index:] pghandler.finish()
def update_getIcon(context): typesToUpdate = { 'Location': ('link_icon.gif', 'link_icon.png'), 'Name': ('document_icon.gif', 'document_icon.png'), 'PleiadesVocabulary': ('folder_icon.gif', 'folder_icon.png'), 'PleiadesVocabularyTerm': ('document_icon.gif', 'document_icon.png'), } catalog = getToolByName(context, 'portal_catalog') logger.info('Updating `getIcon` metadata.') search = catalog.unrestrictedSearchResults _catalog = getattr(catalog, '_catalog', None) getIconPos = None if _catalog is not None: metadata = _catalog.data getIconPos = _catalog.schema.get('getIcon') brains = search(portal_type=typesToUpdate.keys(), sort_on="path") num_objects = len(brains) pghandler = ZLogHandler(1000) pghandler.init('Updating getIcon metadata', num_objects) i = 0 for brain in brains: pghandler.report(i) brain_icon = brain.getIcon old_icon, new_icon = typesToUpdate[brain.portal_type] if brain_icon != new_icon: rid = brain.getRID() record = metadata[rid] new_record = list(record) new_record[getIconPos] = new_icon metadata[rid] = tuple(new_record) i += 1 pghandler.finish() logger.info('Updated `getIcon` metadata.')
def uidDuplications(self): pc = getToolByName(self, 'portal_catalog') ptool = getToolByName(self, 'portal_url') pghandler = ZLogHandler(1000) portal = ptool.getPortalObject() basecat = pc._catalog index = basecat.indexes['UID'] size = len(index._index.keys()) pghandler.init('Remove stale items from catalog', size) cnt = 0 for uid, rid in index._index.items(): path = basecat.paths[rid] pghandler.report(cnt) cnt += 1 try: obj = portal.restrictedTraverse(path) except: # print "could not get obj for path %s, skipping" % path # we ignore this for the moment continue parent = aq_parent(obj) ids = [x['id'] for x in parent._objects] if obj.getId() not in ids and obj.getId() not in parent.objectIds(): print "path %s not in parent " % path del index._index[uid] index._length.change(-1) return "done"
def updateIconsInBrains(context, typesToUpdate=None): """Update getIcon metadata column in given types. typesToUpdate must be a dictionary, for example: { # portal_type: ('old_icon.gif', 'new_icon.png'), 'Document': ('document_icon.gif', 'document_icon.png'), } The portal_types must have an empty icon_expr, because that is the main use case. """ if not typesToUpdate: logger.warn('No typesToUpdate given for updateIconsInBrains.') return catalog = getToolByName(context, 'portal_catalog') logger.info('Updating `getIcon` metadata.') search = catalog.unrestrictedSearchResults _catalog = getattr(catalog, '_catalog', None) getIconPos = None if _catalog is not None: metadata = _catalog.data getIconPos = _catalog.schema.get('getIcon', None) empty_icons = _types_with_empty_icons(context, typesToUpdate) brains = search(portal_type=empty_icons, sort_on='path') num_objects = len(brains) pghandler = ZLogHandler(1000) pghandler.init('Updating getIcon metadata', num_objects) i = 0 for brain in brains: pghandler.report(i) brain_icon = brain.getIcon if not brain_icon: continue if getIconPos is not None: _update_icon_in_single_brain( brain, typesToUpdate, getIconPos, metadata) else: # If we don't have a standard catalog tool, fall back to the # official API obj = brain.getObject() # passing in a valid but inexpensive index, makes sure we don't # reindex the entire catalog including expensive indexes like # SearchableText brain_path = brain.getPath() try: catalog.catalog_object( obj, brain_path, ['id'], True, pghandler) except ConflictError: raise except Exception: pass i += 1 pghandler.finish() logger.info('Updated `getIcon` metadata.')
def refresh_getIcon_catalog_metadata(context): """ get_icon redefined: now boolean: needs to update metadata true if item is type image or has image field (named 'image') e.g. leadimage see https://github.com/plone/Products.CMFPlone/issues/1226 """ # Attention, this relies heavily on catalog internals. # get the more visible zcatalog zcatalog = getToolByName(context, 'portal_catalog') # get the more hidden, inner (real) catalog implementation catalog = zcatalog._catalog try: # Get the getIcon index value and # check if there is a getIcon at all, this may not exist in some # customizations, who knows, but always exists in default Plone metadata_index = catalog.names.index('getIcon') except ValueError: logger.info('`getIcon` not in metadata, skip upgrade step') return cnt = 0 # search whole catalog results = zcatalog.unrestrictedSearchResults() num_objects = len(results) pghandler = ZLogHandler(1000) pghandler.init('Updating getIcon metadata', num_objects) for brain in results: pghandler.report(cnt) # First get the new value try: obj = brain._unrestrictedGetObject() except (AttributeError, KeyError, TypeError, NotFound): continue new_value = bool(getattr(aq_base(obj), 'image', False)) # We can now update the record with the new getIcon value record = list(catalog.data[brain.getRID()]) record[metadata_index] = new_value catalog.data[brain.getRID()] = tuple(record) cnt += 1 # we are curious # done pghandler.finish() logger.info('Reindexed `getIcon` for %s items', str(cnt))
def reindex_mime_type(context): # Attention, this relies heavily on catalog internals. # get the more visible zcatalog zcatalog = getToolByName(context, 'portal_catalog') # get the more hidden, inner (real) catalog implementation catalog = zcatalog._catalog try: metadata_index = catalog.names.index('mime_type') except ValueError: logger.info('`mime_type` not in metadata, skip upgrade step') return # we are interested in dexterity and archtetype images and files: ifaces = ['plone.app.contenttypes.interfaces.IImage', 'plone.app.contenttypes.interfaces.IFile', 'Products.ATContentTypes.interfaces.file.IFileContent', 'Products.ATContentTypes.interfaces.image.IImageContent'] cnt = 0 results = zcatalog.unrestrictedSearchResults(object_provides=ifaces) num_objects = len(results) pghandler = ZLogHandler(1000) pghandler.init('Updating mime_type metadata', num_objects) for brain in results: pghandler.report(cnt) try: obj = brain._unrestrictedGetObject() except (AttributeError, KeyError, TypeError, NotFound): continue if not obj: continue # First get the new value: new_value = '' try: # Dexterity new_value = obj.content_type() except TypeError: # Archetypes new_value = obj.content_type except AttributeError: continue # We can now update the record with the new mime_type value record = list(catalog.data[brain.getRID()]) record[metadata_index] = new_value catalog.data[brain.getRID()] = tuple(record) cnt += 1 pghandler.finish() logger.info('Reindexed `mime_type` for %s items', str(cnt))
def cleanup(self): cat = getToolByName(self, "portal_catalog") purl = getToolByName(self, "portal_url") portal = purl.getPortalObject() zcat = cat._catalog pghandler = ZLogHandler(1000) size = len(zcat.uids) pghandler.init('Remove stale items from catalog', size) cnt = removed = 0 for path in zcat.uids.keys(): pghandler.report(cnt) cnt += 1 try: obj = portal.restrictedTraverse(path) except: # XXX what to do? log.warning("no object found for %s" % path) cat.uncatalog_object(path) removed += 1 log.info('Kill old path: %s' % path) else: if "/".join(obj.getPhysicalPath()) != path: # saved under an old path! kill it cat.uncatalog_object(path) #index = zcat.uids[path] #del zcat.uids[path] #del zcat.paths[index] #zcat._length.change(-1) removed += 1 log.info('Kill old path: %s' % path) else: parent = aq_parent(obj) ids = [x['id'] for x in parent._objects] if obj.getId() not in ids and obj.getId() not in parent.objectIds(): cat.uncatalog_object(path) removed += 1 log.info('ID not found as a child of current parent, kill it: %s' % path) pghandler.finish() log.info('Finished with the catalog, removed a total of %d items' % removed) log.info("Length: %d, len(uids): %d, len(paths): %d" % (zcat._length.value, len(zcat.uids), len(zcat.paths))) transaction.commit()
def addColumn(self, name, default_value=None, threshold=10000): """Adds a row to the meta data schema""" schema = self.schema names = list(self.names) threshold = threshold if threshold is not None else 10000 if name != name.strip(): # Someone could have mistakenly added a space at the end # of the input field. LOG.warning('stripped space from new column %r -> %r', name, name.strip()) name = name.strip() if name in schema: raise CatalogError('The column %s already exists' % name) if name[0] == '_': raise CatalogError('Cannot cache fields beginning with "_"') values = schema.values() if values: schema[name] = max(values) + 1 else: schema[name] = 0 names.append(name) if default_value in (None, ''): default_value = MV if len(self): pghandler = ZLogHandler(threshold) pghandler.init('Adding %s column' % name, len(self)) for i, (key, value) in enumerate(self.data.iteritems()): pghandler.report(i) self.data[key] = value + (default_value, ) pghandler.finish() self.names = tuple(names) self.schema = schema # new column? update the brain self.updateBrains()
def reindex_sortable_title(context): from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE catalog = getToolByName(context, "portal_catalog") _catalog = catalog._catalog indexes = _catalog.indexes sort_title_index = indexes.get("sortable_title", None) if sort_title_index is None: return from Products.PluginIndexes.FieldIndex import FieldIndex if not isinstance(sort_title_index, FieldIndex.FieldIndex): return change = [] pghandler = ZLogHandler(10000) logger.info("Analyzing sortable_title index") pghandler.init("Analyzing sortable_title index", len(sort_title_index)) for i, (name, rids) in enumerate(sort_title_index._index.iteritems()): pghandler.report(i) if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name): if hasattr(rids, "keys"): change.extend(list(rids.keys())) else: change.append(rids) pghandler.finish() update_metadata = "sortable_title" in _catalog.schema pghandler = ZLogHandler(1000) logger.info("Updating sortable_title index") pghandler.init("Updating sortable_title index", len(change)) for i, rid in enumerate(change): pghandler.report(i) brain = _catalog[rid] try: obj = brain.getObject() except (AttributeError, KeyError): continue if update_metadata: obj.reindexObject() else: obj.reindexObject(idxs=["sortable_title"]) pghandler.finish() logger.info("Updated `sortable_title` index.")
def reindex_sortable_title(context): from Products.CMFPlone.CatalogTool import MAX_SORTABLE_TITLE catalog = getToolByName(context, 'portal_catalog') _catalog = catalog._catalog indexes = _catalog.indexes sort_title_index = indexes.get('sortable_title', None) if sort_title_index is None: return from Products.PluginIndexes.FieldIndex import FieldIndex if not isinstance(sort_title_index, FieldIndex.FieldIndex): return change = [] pghandler = ZLogHandler(10000) logger.info('Analyzing sortable_title index') pghandler.init('Analyzing sortable_title index', len(sort_title_index)) for i, (name, rids) in enumerate(sort_title_index._index.iteritems()): pghandler.report(i) if len(name) > MAX_SORTABLE_TITLE or num_sort_regex.match(name): if hasattr(rids, 'keys'): change.extend(list(rids.keys())) else: change.append(rids) pghandler.finish() update_metadata = 'sortable_title' in _catalog.schema pghandler = ZLogHandler(1000) logger.info('Updating sortable_title index') pghandler.init('Updating sortable_title index', len(change)) for i, rid in enumerate(change): pghandler.report(i) brain = _catalog[rid] try: obj = brain.getObject() except (AttributeError, KeyError): continue if update_metadata: obj.reindexObject() else: obj.reindexObject(idxs=['sortable_title']) pghandler.finish() logger.info('Updated `sortable_title` index.')
def agenda_item_project_set_summary_html(context): """Set the mimetype of the summary to text/html. Otherwise when editing you get a bare textarea and cannot save it. """ catalog = getToolByName(context, 'portal_catalog') brains = catalog(portal_type='AgendaItemProject') num_objects = len(brains) logger.info("Updating agenda item summaries.") fixed = 0 pghandler = ZLogHandler(100) pghandler.init('Setting mimetype html for agenda item summaries', num_objects) for index, brain in enumerate(brains): pghandler.report(index) obj = brain.getObject() if obj.summary.mimetype != 'text/html': obj.summary.mimetype = 'text/html' fixed += 1 pghandler.finish() logger.info("Have set summary mimetype to text/html for {0}/{1} agenda " "items.".format(fixed, len(brains)))
def update_catalog_metadata(context, types=None): """Update catalog metadata. Adapted from updateIconMetadata in plone.app.upgrade v40/betas.py. """ catalog = getToolByName(context, 'portal_catalog') search = catalog.unrestrictedSearchResults if types is None: logger.info('Updating catalog metadata.') brains = search(sort_on="path") else: if isinstance(types, basestring): types = [types] logger.info('Updating catalog metadata for %s.', ', '.join(types)) brains = search(portal_type=types, sort_on="path") num_objects = len(brains) # Yes, this logs quite often, but I just know I am going to thank # myself for that... Feedback about progress is good for my heart. # [Maurits] pghandler = ZLogHandler(10) pghandler.init('Updating catalog metadata', num_objects) i = 0 for brain in brains: pghandler.report(i) obj = brain.getObject() # passing in a valid but inexpensive index, makes sure we don't # reindex the entire catalog including expensive indexes like # SearchableText brain_path = brain.getPath() try: catalog.catalog_object(obj, brain_path, ['id'], True, pghandler) except ConflictError: raise except Exception: pass i += 1 pghandler.finish() logger.info('Done updating catalog metadata.')
def fix_themes_navigation(context): """ Add INavigationRoot marker interface for eea.themecentre IThemeTaggable objects. """ ctool = getToolByName(context, 'portal_catalog') brains = ctool.unrestrictedSearchResults( object_provides='eea.themecentre.interfaces.IThemeCentre') logger.info("Fixing eea.themecentre navigation ...") pghandler = ZLogHandler(100) pghandler.init('Fixing eea.themecentre navigation', len(brains)) for index, brain in enumerate(brains): doc = brain.getObject() if not INavigationRoot.providedBy(doc): alsoProvides(doc, INavigationRoot) pghandler.report(index) if index % 100 == 0: transaction.savepoint(optimistic=True) transaction.savepoint(optimistic=True) pghandler.finish() logger.info("Fixing eea.themecentre navigation ... DONE")
def fix_themes_navigation(context): """ Add INavigationRoot marker interface for eea.themecentre IThemeTaggable objects. """ ctool = getToolByName(context, 'portal_catalog') brains = ctool.unrestrictedSearchResults( object_provides='eea.themecentre.interfaces.IThemeCentre') logger.info("Fixing eea.themecentre navigation ...") pghandler = ZLogHandler(100) pghandler.init('Fixing eea.themecentre navigation', len(brains)) for index, brain in enumerate(brains): doc = brain.getObject() if not INavigationRoot.providedBy(doc): alsoProvides(doc, INavigationRoot) pghandler.report(index) if index % 100 == 0: transaction.savepoint(optimistic=True) transaction.savepoint(optimistic=True) pghandler.finish() logger.info("Fixing eea.themecentre navigation ... DONE")
def updateIconMetadata(context): """Update getIcon metadata column for all core content""" catalog = getToolByName(context, 'portal_catalog') logger.info('Updating `getIcon` metadata.') search = catalog.unrestrictedSearchResults _catalog = getattr(catalog, '_catalog', None) getIconPos = None if _catalog is not None: metadata = _catalog.data getIconPos = _catalog.schema.get('getIcon', None) typesToUpdate = { 'Document': ('document_icon.gif', 'document_icon.png'), 'Event': ('event_icon.gif', 'event_icon.png'), 'File': ('file_icon.gif', 'file_icon.png'), 'Folder': ('folder_icon.gif', 'folder_icon.png'), 'Image': ('image_icon.gif', 'image_icon.png'), 'Link': ('link_icon.gif', 'link_icon.png'), 'News Item': ('newsitem_icon.gif', 'newsitem_icon.png'), 'Topic': ('topic_icon.gif', 'topic_icon.png'), } ttool = getToolByName(context, 'portal_types') empty_icons = [] for name in typesToUpdate.keys(): fti = ttool.get(name) if fti: icon_expr = fti.getIconExprObject() if not icon_expr: empty_icons.append(name) brains = search(portal_type=empty_icons, sort_on="path") num_objects = len(brains) pghandler = ZLogHandler(1000) pghandler.init('Updating getIcon metadata', num_objects) i = 0 for brain in brains: pghandler.report(i) brain_icon = brain.getIcon if not brain_icon: continue old_icons = typesToUpdate[brain.portal_type] if getIconPos is not None: # if the old icon is a standard icon, we assume no customization # has taken place and we can simply empty the getIcon metadata # without loading the object new_value = '' if brain_icon not in old_icons: # Otherwise we need to ask the object new_value = '' obj = brain.getObject() method = getattr(aq_base(obj), 'getIcon', None) if method is not None: try: new_value = obj.getIcon if callable(new_value): new_value = new_value() except ConflictError: raise except Exception: new_value = '' if brain_icon != new_value: rid = brain.getRID() record = metadata[rid] new_record = list(record) new_record[getIconPos] = new_value metadata[rid] = tuple(new_record) else: # If we don't have a standard catalog tool, fall back to the # official API obj = brain.getObject() # passing in a valid but inexpensive index, makes sure we don't # reindex the entire catalog including expensive indexes like # SearchableText brain_path = brain.getPath() try: catalog.catalog_object(obj, brain_path, ['id'], True, pghandler) except ConflictError: raise except Exception: pass i += 1 pghandler.finish() logger.info('Updated `getIcon` metadata.')
def updateIconMetadata(context): """Update getIcon metadata column for all core content""" catalog = getToolByName(context, 'portal_catalog') logger.info('Updating `getIcon` metadata.') search = catalog.unrestrictedSearchResults _catalog = getattr(catalog, '_catalog', None) getIconPos = None if _catalog is not None: metadata = _catalog.data getIconPos = _catalog.schema.get('getIcon', None) typesToUpdate = { 'Document' : ('document_icon.gif', 'document_icon.png'), 'Event' : ('event_icon.gif', 'event_icon.png'), 'File' : ('file_icon.gif', 'file_icon.png'), 'Folder' : ('folder_icon.gif', 'folder_icon.png'), 'Image' : ('image_icon.gif', 'image_icon.png'), 'Link' : ('link_icon.gif', 'link_icon.png'), 'News Item' : ('newsitem_icon.gif', 'newsitem_icon.png'), 'Topic' : ('topic_icon.gif', 'topic_icon.png'), } ttool = getToolByName(context, 'portal_types') empty_icons = [] for name in typesToUpdate.keys(): fti = ttool.get(name) if fti: icon_expr = fti.getIconExprObject() if not icon_expr: empty_icons.append(name) brains = search(portal_type=empty_icons, sort_on="path") num_objects = len(brains) pghandler = ZLogHandler(1000) pghandler.init('Updating getIcon metadata', num_objects) i = 0 for brain in brains: pghandler.report(i) brain_icon = brain.getIcon if not brain_icon: continue old_icons = typesToUpdate[brain.portal_type] if getIconPos is not None: # if the old icon is a standard icon, we assume no customization # has taken place and we can simply empty the getIcon metadata # without loading the object new_value = '' if brain_icon not in old_icons: # Otherwise we need to ask the object new_value = '' obj = brain.getObject() method = getattr(aq_base(obj), 'getIcon', None) if method is not None: try: new_value = obj.getIcon if callable(new_value): new_value = new_value() except ConflictError: raise except Exception: new_value = '' if brain_icon != new_value: rid = brain.getRID() record = metadata[rid] new_record = list(record) new_record[getIconPos] = new_value metadata[rid] = tuple(new_record) else: # If we don't have a standard catalog tool, fall back to the # official API obj = brain.getObject() # passing in a valid but inexpensive index, makes sure we don't # reindex the entire catalog including expensive indexes like # SearchableText brain_path = brain.getPath() try: catalog.catalog_object(obj, brain_path, ['id'], True, pghandler) except ConflictError: raise except Exception: pass i += 1 pghandler.finish() logger.info('Updated `getIcon` metadata.')